examples/gpt.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "\n",
    "if 'has_changed_dir' not in globals():\n",
    "    repo_path = os.path.abspath(os.path.join('..'))\n",
    "    \n",
    "    if repo_path not in sys.path:\n",
    "        sys.path.append(repo_path)\n",
    "    \n",
    "    os.chdir(repo_path)\n",
    "    \n",
    "    globals()['has_changed_dir'] = True\n",
    "\n",
    "import math\n",
    "from pathlib import Path\n",
    "from typing import Optional\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from tokenizers.implementations import ByteLevelBPETokenizer\n",
    "from tokenizers.processors import TemplateProcessing\n",
    "from tqdm import tqdm\n",
    "\n",
    "import neunet\n",
    "import neunet.nn as nn\n",
    "from datasets import load_dataset  # type: ignore\n",
    "from neunet import Tensor\n",
    "from neunet.optim import Adam\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = \"cuda\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# [Model implementation]\n",
    "\n",
    "class MultiHeadAttention(nn.Module):\n",
    "    def __init__(self, d_model, n_heads, dropout=0.1):\n",
    "        super(MultiHeadAttention, self).__init__()\n",
    "        self.d_model = d_model\n",
    "        self.n_heads = n_heads\n",
    "        self.scale = math.sqrt(d_model)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "\n",
    "        if d_model % n_heads != 0:\n",
    "            raise ValueError(\"d_model must be divisible by n_heads\")\n",
    "\n",
    "        self.depth = d_model // n_heads\n",
    "\n",
    "        self.wq = nn.Linear(d_model, d_model)\n",
    "        self.wk = nn.Linear(d_model, d_model)\n",
    "        self.wv = nn.Linear(d_model, d_model)\n",
    "\n",
    "        self.fc = nn.Linear(d_model, d_model)\n",
    "\n",
    "\n",
    "    def forward(self, q: Tensor, k: Tensor, v: Tensor, mask: Optional[Tensor]=None):\n",
    "        batch_size = q.shape[0]\n",
    "        q = self.wq(q).contiguous().reshape(batch_size, -1, self.n_heads, self.depth).transpose(0, 2, 1, 3)\n",
    "        k = self.wk(k).contiguous().reshape(batch_size, -1, self.n_heads, self.depth).transpose(0, 2, 1, 3)\n",
    "        v = self.wv(v).contiguous().reshape(batch_size, -1, self.n_heads, self.depth).transpose(0, 2, 1, 3)\n",
    "\n",
    "        scores = neunet.matmul(q, k.transpose(0, 1, 3, 2)) / self.scale\n",
    "        if mask is not None:\n",
    "            mask = mask[:, None, ...]\n",
    "            scores = neunet.where(mask == 0, -1e9, scores)\n",
    "\n",
    "        attn = self.dropout(nn.Softmax(axis = -1)(scores))\n",
    "\n",
    "        x = neunet.matmul(attn, v)\n",
    "        x = x.contiguous().transpose(0, 2, 1, 3).reshape(batch_size, -1, self.n_heads * self.depth)\n",
    "        x = self.fc(x)\n",
    "\n",
    "        return x, attn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PositionwiseFeedForward(nn.Module):\n",
    "    def __init__(self, d_model, d_ff, dropout=0.1):\n",
    "        super(PositionwiseFeedForward, self).__init__()\n",
    "        self.fc_1 = nn.Linear(d_model, d_ff)\n",
    "        self.fc_2 = nn.Linear(d_ff, d_model)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "\n",
    "        self.activation = nn.Swish()\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.fc_1(x)\n",
    "        x = self.activation(x)\n",
    "        x = self.dropout(x)\n",
    "        x = self.fc_2(x)\n",
    "\n",
    "        return x\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DecoderLayer(nn.Module):\n",
    "    def __init__(self, d_model, n_heads, d_ff, dropout=0.1):\n",
    "        super(DecoderLayer, self).__init__()\n",
    "        self.self_attn = MultiHeadAttention(d_model, n_heads, dropout)\n",
    "        self.cross_attn = MultiHeadAttention(d_model, n_heads, dropout)\n",
    "        self.ffn = PositionwiseFeedForward(d_model, d_ff, dropout)\n",
    "        self.norm1 = nn.RMSNorm(d_model)\n",
    "        self.norm2 = nn.RMSNorm(d_model)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "\n",
    "    def forward(self, x, mask):\n",
    "        # Masked self-attention (for the target sequence)\n",
    "        nx1 = self.norm1(x)\n",
    "        _x, attn = self.self_attn(nx1, nx1, nx1, mask)\n",
    "        x = x + self.dropout(_x)\n",
    "        \n",
    "        # Feed-forward network\n",
    "        nx2 = self.norm2(x)\n",
    "        _x = self.ffn(nx2)\n",
    "        x = x + self.dropout(_x)\n",
    "        \n",
    "        return x, attn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PositionalEncoding(nn.Module):\n",
    "    def __init__(self, d_model, max_len=5000):\n",
    "        \"\"\"\n",
    "        Inputs\n",
    "            d_model - Hidden dimensionality of the input.\n",
    "            max_len - Maximum length of a sequence to expect.\n",
    "            https://uvadlc-notebooks.readthedocs.io/en/latest/tutorial_notebooks/tutorial6/Transformers_and_MHAttention.html\n",
    "        \"\"\"\n",
    "        super(PositionalEncoding, self).__init__()\n",
    "        pe = neunet.zeros(max_len, d_model, requires_grad=False)\n",
    "        position = neunet.arange(0, max_len, dtype=neunet.float32)[:, None, ...]\n",
    "        div_term = neunet.exp(neunet.arange(0, d_model, 2) * (-math.log(10000.0) / d_model))\n",
    "        pe[:, 0::2] = neunet.sin(position * div_term)\n",
    "        pe[:, 1::2] = neunet.cos(position * div_term)\n",
    "\n",
    "        self.pe = pe[None, ...]\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x + self.pe[:, :x.shape[1]] # (batch_size, seq_len, d_model)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Decoder(nn.Module):\n",
    "    def __init__(self, tgt_vocab_size, d_model, n_heads, d_ff, n_layers, dropout=0.1, max_len=5000):\n",
    "        super(Decoder, self).__init__()\n",
    "        self.token_embedding = nn.Embedding(tgt_vocab_size, d_model)\n",
    "        self.position_embedding = PositionalEncoding(d_model, max_len)\n",
    "        self.layers = nn.ModuleList([DecoderLayer(d_model, n_heads, d_ff, dropout) for _ in range(n_layers)])\n",
    "\n",
    "        self.fc_out = nn.Linear(d_model, tgt_vocab_size)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "\n",
    "        self.scale = math.sqrt(d_model)\n",
    "\n",
    "\n",
    "    def forward(self, x, mask):\n",
    "        x = self.token_embedding(x) * self.scale\n",
    "        x = self.position_embedding(x)\n",
    "        x = self.dropout(x)\n",
    "        for layer in self.layers:\n",
    "            x, attn = layer(x, mask)\n",
    "\n",
    "        out = self.fc_out(x)\n",
    "        return out, attn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GPT(nn.Module):\n",
    "    def __init__(self, decoder: Decoder, pad_idx) -> None:\n",
    "        self.decoder = decoder\n",
    "        self.pad_idx = pad_idx\n",
    "\n",
    "    def get_pad_mask(self, x):\n",
    "        #x: (batch_size, seq_len)\n",
    "        return (x != self.pad_idx).astype(int)[:, np.newaxis, :]\n",
    "\n",
    "    def get_sub_mask(self, x):\n",
    "        #x: (batch_size, seq_len)\n",
    "        seq_len = x.shape[1]\n",
    "        subsequent_mask = np.triu(np.ones((seq_len, seq_len)), k = 1).astype(int)\n",
    "        subsequent_mask = np.logical_not(subsequent_mask)\n",
    "        return subsequent_mask\n",
    "\n",
    "    def forward(self, x) -> tuple[Tensor, Tensor]:\n",
    "        # x: (batch_size, target_seq_len)\n",
    "        # mask: (batch_size, seq_len, seq_len)\n",
    "\n",
    "        mask = self.get_pad_mask(x) & self.get_sub_mask(x)\n",
    "\n",
    "        x, mask = neunet.tensor(x, dtype=neunet.int32, device=device), neunet.tensor(mask, dtype=neunet.int32, device=device)\n",
    "        \n",
    "\n",
    "        out, attention = self.decoder(x, mask)\n",
    "        # output: (batch_size, target_seq_len, vocab_size)\n",
    "        # attn: (batch_size, heads_num, target_seq_len, source_seq_len)\n",
    "        return out, attention\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# [Data preprocessing]\n",
    "\n",
    "BATCH_SIZE = 4\n",
    "\n",
    "PAD_TOKEN = '<pad>' # noqa: S105\n",
    "SOS_TOKEN = '<sos>' # noqa: S105\n",
    "EOS_TOKEN = '<eos>' # noqa: S105\n",
    "\n",
    "\n",
    "DATASET_PATH = Path(\"./datasets/sd-prompts/\")\n",
    "SAVE_PATH = Path(\"./saved models/gpt/\")\n",
    "\n",
    "\n",
    "if not DATASET_PATH.exists():\n",
    "    data = load_dataset(\"Gustavosta/Stable-Diffusion-Prompts\", cache_dir=\"datasets/sd-prompts\")\n",
    "\n",
    "    for split, split_dataset in data.items():\n",
    "\n",
    "        with Path(f\"./datasets/sd-prompts/sd-prompts-{split}.txt\").open('w', encoding='utf-8') as f:\n",
    "            for item in split_dataset:\n",
    "                f.write(item['Prompt'] + '\\n')\n",
    "\n",
    "\n",
    "FILE_PATHS = [DATASET_PATH / \"sd-prompts-train.txt\", DATASET_PATH / \"sd-prompts-test.txt\"]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# [Train and load Tokenizer]\n",
    "\n",
    "if not (SAVE_PATH / \"vocab\").exists():\n",
    "    tokenizer = ByteLevelBPETokenizer()\n",
    "\n",
    "    tokenizer.train(files=[str(path) for path in FILE_PATHS], vocab_size=15000, min_frequency=1, special_tokens=[\n",
    "        PAD_TOKEN,\n",
    "        SOS_TOKEN,\n",
    "        EOS_TOKEN,\n",
    "        # UNK_TOKEN\n",
    "    ])\n",
    "\n",
    "    (SAVE_PATH / \"vocab\").mkdir(parents=True)\n",
    "    tokenizer.save_model(str(SAVE_PATH / \"vocab\"), \"sd-prompts-tokenizer\")\n",
    "\n",
    "tokenizer = ByteLevelBPETokenizer(\n",
    "    str(SAVE_PATH / \"vocab/sd-prompts-tokenizer-vocab.json\"),\n",
    "    str(SAVE_PATH / \"vocab/sd-prompts-tokenizer-merges.txt\"),\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "PAD_INDEX = tokenizer.token_to_id(PAD_TOKEN)\n",
    "SOS_INDEX = tokenizer.token_to_id(SOS_TOKEN)\n",
    "EOS_INDEX = tokenizer.token_to_id(EOS_TOKEN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing datasets\\sd-prompts\\sd-prompts-train.txt\n",
      "Processing datasets\\sd-prompts\\sd-prompts-test.txt\n"
     ]
    }
   ],
   "source": [
    "\n",
    "class DataPreprocessor():\n",
    "    def __init__(self, tokenizer: ByteLevelBPETokenizer):\n",
    "        self.tokenizer = tokenizer\n",
    "\n",
    "        self.tokenizer._tokenizer.post_processor  = TemplateProcessing( # noqa SLF001\n",
    "            single=f\"{SOS_TOKEN} $A {EOS_TOKEN}\",\n",
    "            special_tokens=[\n",
    "                (f\"{SOS_TOKEN}\", tokenizer.token_to_id(f\"{SOS_TOKEN}\")),\n",
    "                (f\"{EOS_TOKEN}\", tokenizer.token_to_id(f\"{EOS_TOKEN}\")),\n",
    "            ],\n",
    "        )\n",
    "\n",
    "        # self.tokenizer.enable_truncation(max_length=151)\n",
    "        self.tokenizer.enable_padding(pad_token = PAD_TOKEN)\n",
    "        \n",
    "    def tokenize(self, paths: list[str], batch_size: int, lines_limit: Optional[int] = None) -> list[np.ndarray]:\n",
    "        examples = []\n",
    "\n",
    "        for src_file in paths:\n",
    "            print(f\"Processing {src_file}\")\n",
    "            path_src_file = Path(src_file)\n",
    "            lines = path_src_file.read_text(encoding=\"utf-8\").splitlines()\n",
    "\n",
    "            if lines_limit:\n",
    "                lines = lines[:lines_limit]\n",
    "\n",
    "            for i in range(0, len(lines), batch_size):\n",
    "                examples.append(np.array([x.ids for x in self.tokenizer.encode_batch(lines[i:i+batch_size])]))\n",
    "\n",
    "\n",
    "        return examples\n",
    "\n",
    "    def __call__(self, paths: list[str], batch_size: int, lines_limit: Optional[int] = None) -> list[np.ndarray]:\n",
    "        return self.tokenize(paths, batch_size, lines_limit)\n",
    "\n",
    "data_post_processor = DataPreprocessor(tokenizer)\n",
    "\n",
    "train_data = data_post_processor([str(FILE_PATHS[0])], batch_size = BATCH_SIZE, lines_limit=20000)\n",
    "val_data = data_post_processor([str(FILE_PATHS[1])], batch_size = BATCH_SIZE, lines_limit=2000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# [Model intialization]\n",
    "\n",
    "decoder = Decoder(\n",
    "    tgt_vocab_size = tokenizer.get_vocab_size(),\n",
    "    d_model = 512,\n",
    "    n_heads = 8,\n",
    "    d_ff = 2048,\n",
    "    n_layers = 8,\n",
    "    dropout = 0.1\n",
    ")\n",
    "\n",
    "\n",
    "model = GPT(\n",
    "    decoder = decoder,\n",
    "    pad_idx = tokenizer.token_to_id(PAD_TOKEN)\n",
    ")\n",
    "\n",
    "model = model.to(device)\n",
    "\n",
    "optimizer = Adam(model.parameters(), lr=1.5e-4, betas=(0.9, 0.98), eps = 1e-9)\n",
    "\n",
    "loss_function = nn.CrossEntropyLoss(ignore_index = PAD_INDEX)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# [train, eval, predict methods definition]\n",
    "\n",
    "def train_step(dataset: list[np.ndarray], epoch: int, epochs: int) -> float:\n",
    "    total_loss = 0.0\n",
    "    model.train()\n",
    "\n",
    "    tqdm_range = tqdm(enumerate(dataset), total = len(dataset))\n",
    "    for batch_num, batch in tqdm_range:\n",
    "\n",
    "        output, _ = model.forward(batch[:,:-1])\n",
    "        \n",
    "        output = output.reshape(output.shape[0] * output.shape[1], output.shape[2])\n",
    "\n",
    "        loss = loss_function(output, neunet.tensor(batch[:, 1:].flatten(), device=device, dtype=neunet.int32))\n",
    "        loss.backward()\n",
    "\n",
    "        optimizer.step()\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        total_loss += loss.detach().item()\n",
    "\n",
    "\n",
    "        tqdm_range.set_description(\n",
    "                f\"training | loss: {loss.detach().item():.7f} | perplexity: {np.exp(loss.detach().item()):.7f} | epoch {epoch + 1}/{epochs}\" #loss: {loss:.4f}\n",
    "            )\n",
    "\n",
    "        if batch_num == (len(dataset) - 1):\n",
    "            epoch_loss = total_loss / len(dataset)\n",
    "\n",
    "            tqdm_range.set_description(\n",
    "                    f\"training | avg loss: {epoch_loss:.7f} | avg perplexity: {np.exp(epoch_loss):.7f} | epoch {epoch + 1}/{epochs}\"\n",
    "            )\n",
    "\n",
    "    return epoch_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def eval(dataset: list[np.ndarray]) -> float:\n",
    "    total_loss = 0.0\n",
    "    model.eval()\n",
    "\n",
    "    tqdm_range = tqdm(enumerate(dataset), total = len(dataset))\n",
    "    for batch_num, batch in tqdm_range:\n",
    "        \n",
    "        output, _ = model.forward(batch[:,:-1])\n",
    "        \n",
    "        output = output.reshape(output.shape[0] * output.shape[1], output.shape[2])\n",
    "        \n",
    "        loss = loss_function(output, neunet.tensor(batch[:, 1:].flatten(), device=device, dtype=neunet.int32))\n",
    "        total_loss += loss.detach().item()\n",
    "        \n",
    "        tqdm_range.set_description(\n",
    "                f\"testing  | loss: {loss.detach().item():.7f} | perplexity: {np.exp(loss.detach().item()):.7f}\"\n",
    "            )\n",
    "\n",
    "        if batch_num == (len(dataset) - 1):\n",
    "            epoch_loss = total_loss / len(dataset)\n",
    "\n",
    "            tqdm_range.set_description(\n",
    "                    f\"testing  | avg loss: {epoch_loss:.7f} | avg perplexity: {np.exp(epoch_loss):.7f}\"\n",
    "            )\n",
    "\n",
    "    return epoch_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(train_data: list[np.ndarray], val_data: list[np.ndarray], epochs: int, save_every_epochs: int, save_path: Optional[str] = None, validation_check: bool = False):\n",
    "    best_val_loss = float('inf')\n",
    "    \n",
    "    train_loss_history = []\n",
    "    val_loss_history = []\n",
    "\n",
    "    for epoch in range(epochs):\n",
    "\n",
    "        train_loss_history.append(train_step(train_data, epoch, epochs))\n",
    "        val_loss_history.append(eval(val_data))\n",
    "\n",
    "\n",
    "        if (save_path is not None) and ((epoch + 1) % save_every_epochs == 0):\n",
    "            if not Path(save_path).exists():\n",
    "                Path(save_path).mkdir(parents=True, exist_ok=True)\n",
    "            if validation_check == False:\n",
    "\n",
    "                neunet.save(model.state_dict(), f\"{save_path}/gpt_{epoch + 1}.nt\")\n",
    "            else:\n",
    "                if val_loss_history[-1] < best_val_loss:\n",
    "                    best_val_loss = val_loss_history[-1]\n",
    "                    \n",
    "                    neunet.save(model.state_dict(), f\"{save_path}/gpt_{epoch + 1}.nt\")\n",
    "                else:\n",
    "                    print('Current validation loss is higher than previous. Not saved.')\n",
    "                    break\n",
    "            \n",
    "    return train_loss_history, val_loss_history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(sentence: str = \"\", max_length: int = 50, temperature: float = 0.7) -> tuple[str, Tensor]:\n",
    "    model.eval()\n",
    "\n",
    "    tokens: list = [SOS_INDEX, *tokenizer.encode(sentence, add_special_tokens=False).ids]\n",
    "   \n",
    "    for _ in range(max_length):\n",
    "        inputs = np.asarray(tokens).reshape(1, -1)\n",
    "\n",
    "        outputs, attention = model.forward(inputs)\n",
    "\n",
    "        outputs = outputs[:, -1] / temperature\n",
    "        outputs = nn.Softmax(axis=-1)(outputs)\n",
    "\n",
    "        probs = outputs.reshape(-1).detach().cpu().numpy()\n",
    "\n",
    "        next_token = np.random.choice(len(probs), p=probs)\n",
    "        \n",
    "        # next_token = outputs.detach().cpu().numpy().argmax(axis=-1)[:, -1].item() # eq. temperature = 0.0\n",
    "\n",
    "        tokens.append(next_token)\n",
    "\n",
    "        if next_token == EOS_INDEX or len(tokens) >= max_length:\n",
    "            break\n",
    "    \n",
    "    # Remove special tokens\n",
    "    if SOS_INDEX in tokens:\n",
    "        tokens.remove(SOS_INDEX)\n",
    "    if EOS_INDEX in tokens:\n",
    "        tokens.remove(EOS_INDEX)\n",
    "\n",
    "    decoded_sentence = tokenizer.decode(tokens)\n",
    "\n",
    "    return decoded_sentence, attention\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training | avg loss: 9.2024686 | avg perplexity: 9921.5908248 | epoch 1/30: 100%|██████████| 5000/5000 [12:49<00:00,  6.50it/s]            \n",
      "testing  | avg loss: 4.6352218 | avg perplexity: 103.0507714: 100%|██████████| 500/500 [00:25<00:00, 19.41it/s]\n",
      "training | avg loss: 3.7912031 | avg perplexity: 44.3096772 | epoch 2/30: 100%|██████████| 5000/5000 [12:41<00:00,  6.57it/s]\n",
      "testing  | avg loss: 3.2742608 | avg perplexity: 26.4236849: 100%|██████████| 500/500 [00:25<00:00, 19.46it/s]\n",
      "training | avg loss: 3.1931661 | avg perplexity: 24.3654491 | epoch 3/30: 100%|██████████| 5000/5000 [12:55<00:00,  6.44it/s]\n",
      "testing  | avg loss: 3.0415197 | avg perplexity: 20.9370372: 100%|██████████| 500/500 [00:25<00:00, 19.38it/s]\n",
      "training | avg loss: 2.9606324 | avg perplexity: 19.3101797 | epoch 4/30: 100%|██████████| 5000/5000 [12:42<00:00,  6.55it/s]\n",
      "testing  | avg loss: 2.8808553 | avg perplexity: 17.8295162: 100%|██████████| 500/500 [00:24<00:00, 20.21it/s]\n",
      "training | avg loss: 2.7897262 | avg perplexity: 16.2765624 | epoch 5/30: 100%|██████████| 5000/5000 [12:34<00:00,  6.62it/s]\n",
      "testing  | avg loss: 2.7932064 | avg perplexity: 16.3333077: 100%|██████████| 500/500 [00:25<00:00, 19.50it/s]\n",
      "training | avg loss: 2.6519337 | avg perplexity: 14.1814344 | epoch 6/30: 100%|██████████| 5000/5000 [12:33<00:00,  6.64it/s]\n",
      "testing  | avg loss: 2.7174952 | avg perplexity: 15.1423464: 100%|██████████| 500/500 [00:25<00:00, 19.67it/s]\n",
      "training | avg loss: 2.5374272 | avg perplexity: 12.6470911 | epoch 7/30: 100%|██████████| 5000/5000 [12:37<00:00,  6.60it/s]\n",
      "testing  | avg loss: 2.6616154 | avg perplexity: 14.3194017: 100%|██████████| 500/500 [00:24<00:00, 20.12it/s]\n",
      "training | avg loss: 2.4395581 | avg perplexity: 11.4679720 | epoch 8/30: 100%|██████████| 5000/5000 [12:38<00:00,  6.59it/s]\n",
      "testing  | avg loss: 2.6264456 | avg perplexity: 13.8245443: 100%|██████████| 500/500 [00:25<00:00, 19.60it/s]\n",
      "training | avg loss: 2.3631945 | avg perplexity: 10.6248385 | epoch 9/30: 100%|██████████| 5000/5000 [12:36<00:00,  6.61it/s]\n",
      "testing  | avg loss: 2.6067684 | avg perplexity: 13.5551748: 100%|██████████| 500/500 [00:25<00:00, 19.39it/s]\n",
      "training | avg loss: 2.3212453 | avg perplexity: 10.1883543 | epoch 10/30: 100%|██████████| 5000/5000 [12:36<00:00,  6.61it/s]\n",
      "testing  | avg loss: 2.5969647 | avg perplexity: 13.4229329: 100%|██████████| 500/500 [00:25<00:00, 19.25it/s]\n",
      "training | avg loss: 2.2753301 | avg perplexity: 9.7311304 | epoch 11/30: 100%|██████████| 5000/5000 [12:30<00:00,  6.66it/s]\n",
      "testing  | avg loss: 2.5856186 | avg perplexity: 13.2714960: 100%|██████████| 500/500 [00:25<00:00, 19.90it/s]\n",
      "training | avg loss: 2.2190665 | avg perplexity: 9.1987395 | epoch 12/30: 100%|██████████| 5000/5000 [12:38<00:00,  6.60it/s]\n",
      "testing  | avg loss: 2.5722157 | avg perplexity: 13.0948070: 100%|██████████| 500/500 [00:25<00:00, 19.49it/s]\n",
      "training | avg loss: 2.1687687 | avg perplexity: 8.7475066 | epoch 13/30: 100%|██████████| 5000/5000 [12:30<00:00,  6.66it/s]\n",
      "testing  | avg loss: 2.5500583 | avg perplexity: 12.8078500: 100%|██████████| 500/500 [00:24<00:00, 20.06it/s]\n",
      "training | avg loss: 2.1236505 | avg perplexity: 8.3616055 | epoch 14/30: 100%|██████████| 5000/5000 [12:29<00:00,  6.67it/s]\n",
      "testing  | avg loss: 2.5578914 | avg perplexity: 12.9085691: 100%|██████████| 500/500 [00:24<00:00, 20.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current validation loss is higher than previous. Not saved.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# [Train the Model]\n",
    "\n",
    "# model.load_state_dict(neunet.load(\"./saved models/gpt/gpt_2.nt\"))\n",
    "\n",
    "train_loss_history, val_loss_history = None, None\n",
    "train_loss_history, val_loss_history = train(train_data, val_data, epochs=30, save_every_epochs=1, save_path = str(SAVE_PATH), validation_check=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAioAAAHHCAYAAACRAnNyAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAABNWElEQVR4nO3deXhTZd4+8PtkX5qmC3STLmxS9pF1EFQUZJURXPiJVUGdcdSiML446uuLoohV3BidGQbHGWQccB1R1EEERhlBERBQQHahLbSlQNukbZqlyfn9cZq0aUtp0yQnae/PdZ0rycnJOd9GbW+f8yyCKIoiiIiIiCKQQu4CiIiIiC6EQYWIiIgiFoMKERERRSwGFSIiIopYDCpEREQUsRhUiIiIKGIxqBAREVHEYlAhIiKiiMWgQkRERBGLQYWIZDdnzhzExMS06lhBELBo0aLQFkREEYNBhagDe/PNNyEIAnbt2iV3KbJas2YNli1bJncZRBQAldwFEBG1RU1NDVSqtv3qWrNmDfbv34/58+eHpigiChkGFSKKKjqdTu4SAAC1tbXweDzQaDRyl0LUofHWDxFhz549mDx5MmJjYxETE4Nx48Zh+/btfse4XC489dRT6N27N3Q6HRITEzFmzBhs3LjRd0xJSQnuvPNOdOvWDVqtFqmpqbj++utx8uTJVtVx+vRpTJ8+HTExMejatSsWLFgAt9vtd0zjPiqVlZWYP38+srKyoNVqkZSUhGuvvRa7d+8GAIwdOxafffYZ8vPzIQgCBEFAVlaW7/OlpaW4++67kZycDJ1Oh8GDB2PVqlV+1zx58iQEQcCLL76IZcuWoWfPntBqtdixYweMRiPmzZvX5Gc5deoUlEol8vLyWvWzE1Hz2KJC1MkdOHAAV1xxBWJjY/H73/8earUaK1aswNixY7FlyxaMHDkSALBo0SLk5eXh17/+NUaMGAGr1Ypdu3Zh9+7duPbaawEAN954Iw4cOIAHHngAWVlZKC0txcaNG1FQUOAXDprjdrsxceJEjBw5Ei+++CI2bdqEl156CT179sR99913wc/de++9+OCDDzB37lz069cP58+fx9atW3Hw4EEMGTIEjz/+OCwWC06dOoVXXnkFAHwdd2tqajB27FgcO3YMc+fORffu3fH+++9jzpw5qKioaBJAVq5cCbvdjnvuuQdarRYZGRmYMWMG3n33Xbz88stQKpW+Y99++22IooicnJw2/zMhogZEIuqwVq5cKQIQd+7cecFjpk+fLmo0GvH48eO+fUVFRaLJZBKvvPJK377BgweLU6dOveB5ysvLRQDiCy+80OY6Z8+eLQIQn376ab/9l112mTh06FC/fQDEJ5980vfabDaLubm5LZ5/6tSpYmZmZpP9y5YtEwGI//znP337nE6nOGrUKDEmJka0Wq2iKIriiRMnRABibGysWFpa6neODRs2iADE9evX++0fNGiQeNVVV7VYFxFdHG/9EHVibrcbX3zxBaZPn44ePXr49qempuLWW2/F1q1bYbVaAQBxcXE4cOAAjh492uy59Ho9NBoNvvrqK5SXlwdUz7333uv3+oorrsDPP//c4mfi4uLw3XffoaioqM3X+/e//42UlBTMmjXLt0+tVuPBBx9EVVUVtmzZ4nf8jTfeiK5du/rtGz9+PNLS0rB69Wrfvv379+PHH3/Ebbfd1uaaiMgfgwpRJ3b27FnYbDb06dOnyXt9+/aFx+NBYWEhAODpp59GRUUFLr30UgwcOBAPP/wwfvzxR9/xWq0Wzz//PNavX4/k5GRceeWVWLp0KUpKSlpVi06naxIC4uPjLxp6li5div379yM9PR0jRozAokWLLhpuvPLz89G7d28oFP6/Cvv27et7v6Hu3bs3OYdCoUBOTg4++ugj2Gw2AMDq1auh0+lw8803t6oOIrowBhUiapUrr7wSx48fx9///ncMGDAAb7zxBoYMGYI33njDd8z8+fNx5MgR5OXlQafTYeHChejbty/27Nlz0fM37N/RFjNnzsTPP/+M1157DWlpaXjhhRfQv39/rF+/PqDztUSv1ze7/4477kBVVRU++ugjiKKINWvW4LrrroPZbA56DUSdDYMKUSfWtWtXGAwGHD58uMl7hw4dgkKhQHp6um9fQkIC7rzzTrz99tsoLCzEoEGDmswS27NnT/zP//wPvvjiC+zfvx9OpxMvvfRSSH+O1NRU3H///fjoo49w4sQJJCYmYsmSJb73BUFo9nOZmZk4evQoPB6P3/5Dhw753m+NAQMG4LLLLsPq1avx9ddfo6CgALfffnuAPw0RNcSgQtSJKZVKTJgwAR9//LHfEOIzZ85gzZo1GDNmDGJjYwEA58+f9/tsTEwMevXqBYfDAQCw2Wyw2+1+x/Ts2RMmk8l3TLC53W5YLBa/fUlJSUhLS/O7ptFobHIcAEyZMgUlJSV49913fftqa2vx2muvISYmBldddVWra7n99tvxxRdfYNmyZUhMTMTkyZMD+ImIqDEOTybqBP7+97/j888/b7J/3rx5eOaZZ7Bx40aMGTMG999/P1QqFVasWAGHw4GlS5f6ju3Xrx/Gjh2LoUOHIiEhAbt27fINCwaAI0eOYNy4cZg5cyb69esHlUqFtWvX4syZM7jllltC8nNVVlaiW7duuOmmmzB48GDExMRg06ZN2Llzp18rztChQ/Huu+/ioYcewvDhwxETE4Np06bhnnvuwYoVKzBnzhx8//33yMrKwgcffIBt27Zh2bJlMJlMra7l1ltvxe9//3usXbsW9913H9RqdSh+ZKLOR+5hR0QUOt7hyRfaCgsLRVEUxd27d4sTJ04UY2JiRIPBIF599dXiN99843euZ555RhwxYoQYFxcn6vV6MTs7W1yyZInodDpFURTFc+fOibm5uWJ2drZoNBpFs9ksjhw5UnzvvfcuWufs2bNFo9HYZP+TTz4pNv41hQbDkx0Oh/jwww+LgwcPFk0mk2g0GsXBgweLf/7zn/0+U1VVJd56661iXFycCMBvqPKZM2fEO++8U+zSpYuo0WjEgQMHiitXrvT7vHd48sWGXk+ZMkUE0OS7I6LACaIoivJEJCKijmXGjBnYt28fjh07JncpRB0G+6gQEQVBcXExPvvsM3aiJQoy9lEhImqHEydOYNu2bXjjjTegVqvx29/+Vu6SiDoUtqgQEbXDli1bcPvtt+PEiRNYtWoVUlJS5C6JqENhHxUiIiKKWGxRISIioojFoEJEREQRK6o703o8HhQVFcFkMl1wimwiIiKKLKIoorKyEmlpaU0WBW0sqoNKUVGR3zokREREFD0KCwvRrVu3Fo+J6qDind66sLDQtx4JERERRTar1Yr09PRWLVMR1UHFe7snNjaWQYWIiCjKtKbbBjvTEhERUcRiUCEiIqKIxaBCREREESuq+6i0ltvthsvlkruMqKRWq6FUKuUug4iIOqkOHVREUURJSQkqKirkLiWqxcXFISUlhXPVEBFR2HXooOINKUlJSTAYDPxD20aiKMJms6G0tBQAkJqaKnNFRETU2XTYoOJ2u30hJTExUe5yopZerwcAlJaWIikpibeBiIgorDpsZ1pvnxSDwSBzJdHP+x2ynw8REYVbhw0qXrzd0378DomISC4dPqgQERFR9GJQ6eCysrKwbNkyucsgIiIKSIftTBvNxo4di1/84hdBCRg7d+6E0Whsf1FEREQyYFBphiiKqHWLECFCo4q8US6iKMLtdkOluvg/vq5du4ahIiIiotDgrZ9mlFU7cbDEiqIKe9ivPWfOHGzZsgV/+MMfIAgCBEHAm2++CUEQsH79egwdOhRarRZbt27F8ePHcf311yM5ORkxMTEYPnw4Nm3a5He+xrd+BEHAG2+8gRkzZsBgMKB3795Yt25dmH9KIiKi1ulUQUUURdictRfdaj0e2F1uWOyuVh1/sU0UxVbX+Ic//AGjRo3Cb37zGxQXF6O4uBjp6ekAgEcffRTPPfccDh48iEGDBqGqqgpTpkzB5s2bsWfPHkyaNAnTpk1DQUFBi9d46qmnMHPmTPz444+YMmUKcnJyUFZW1q7vloiIKBQ61a2fGpcb/Z7YEPbr/vT0RBg0rfuqzWYzNBoNDAYDUlJSAACHDh0CADz99NO49tprfccmJCRg8ODBvteLFy/G2rVrsW7dOsydO/eC15gzZw5mzZoFAHj22Wfx6quvYseOHZg0aVKbfzYiIqJQ6lQtKtFu2LBhfq+rqqqwYMEC9O3bF3FxcYiJicHBgwcv2qIyaNAg33Oj0YjY2FjfNPlERESRpFO1qOjVSvz09MRWHXukpBJOtwfduxhh1Lbva9Krg9Mht/HonQULFmDjxo148cUX0atXL+j1etx0001wOp0tnketVvu9FgQBHo8nKDUSEREFU6cKKoIgtPoWTKxejSpHLVQKRas/EywajQZut/uix23btg1z5szBjBkzAEgtLCdPngxxdUREROEj662fyspKzJ8/H5mZmdDr9bj88suxc+dOOUvy0aikr8bpDn9LQ1ZWFr777jucPHkS586du2BrR+/evfHhhx9i7969+OGHH3DrrbeyZYSIiDoUWYPKr3/9a2zcuBFvvfUW9u3bhwkTJmD8+PE4ffq0nGUBaBBUasP/h3/BggVQKpXo168funbtesE+Jy+//DLi4+Nx+eWXY9q0aZg4cSKGDBkS5mqJiIhCRxDbMnY2iGpqamAymfDxxx9j6tSpvv1Dhw7F5MmT8cwzz1z0HFarFWazGRaLBbGxsX7v2e12nDhxAt27d4dOp2tzfRU2JwrKbDBoVOiVFNPmz3ck7f0uiYiIGmrp73djsrWo1NbWwu12N/nDp9frsXXrVpmqqqeVsUWFiIiIJLIFFZPJhFGjRmHx4sUoKiqC2+3GP//5T3z77bcoLi5u9jMOhwNWq9VvCxV1XVCp9Xjg9sjS6ERERNTpydpH5a233oIoirjkkkug1Wrx6quvYtasWVAomi8rLy8PZrPZt3lnbA0FlUIBpUIAIE+HWiIiIpI5qPTs2RNbtmxBVVUVCgsLsWPHDrhcLvTo0aPZ4x977DFYLBbfVlhYGNL65OxQS0RERBEyj4rRaITRaER5eTk2bNiApUuXNnucVquFVqsNW11apQI1cDOoEBERyUTWoLJhwwaIoog+ffrg2LFjePjhh5GdnY0777xTzrJ82KJCREQkL1lv/VgsFuTm5iI7Oxt33HEHxowZgw0bNjSZ4l0uck76RkRERDK3qMycORMzZ86Us4QWaZTSGj1sUSEiIpIHV09uQcMWFZnmxSMiIurUGFRaoFYKEAQBoijCFUW3f7KysrBs2TK5yyAiImo3BpUWCIIAjZIdaomIiOTCoHIR7FBLREQkHwaVi/AGFUeYWlRef/11pKWlwePxv97111+Pu+66C8ePH8f111+P5ORkxMTEYPjw4di0aVNYaiMiIgq3zhVURBFwVrdp03hqILhscNVUtfmzvq0NHXFvvvlmnD9/Hl9++aVvX1lZGT7//HPk5OSgqqoKU6ZMwebNm7Fnzx5MmjQJ06ZNQ0FBQSi+MSIiIllFxMy0YeOyAc+mtekjXeu2dvnfIkBjbNWh8fHxmDx5MtasWYNx48YBAD744AN06dIFV199NRQKBQYPHuw7fvHixVi7di3WrVuHuXPntrdSIiKiiNK5WlSiRE5ODv71r3/B4XAAAFavXo1bbrkFCoUCVVVVWLBgAfr27Yu4uDjExMTg4MGDbFEhIqIOqXO1qKgNUutGG3g8Ig4UWwEAfVNMUCkDyHZqQ5sOnzZtGkRRxGeffYbhw4fj66+/xiuvvAIAWLBgATZu3IgXX3wRvXr1gl6vx0033QSn09n2uoiIiCJc5woqgtDqWzBeCgBKnQe1bg+cSj1UmtB/ZTqdDjfccANWr16NY8eOoU+fPhgyZAgAYNu2bZgzZw5mzJgBAKiqqsLJkydDXhMREZEcOldQCZBWqZCCSq0HBk14rpmTk4PrrrsOBw4cwG233ebb37t3b3z44YeYNm0aBEHAwoULm4wQIiIi6ijYR6UV5FhF+ZprrkFCQgIOHz6MW2+91bf/5ZdfRnx8PC6//HJMmzYNEydO9LW2EBERdTRsUWkFOSZ9UygUKCpq2p8mKysL//nPf/z25ebm+r3mrSAiIuoo2KLSCnK0qBARERGDSqtwvR8iIiJ5MKi0grdFxeX2wNOGWWaJiIiofRhUWkGlEKAQBIgAXGxVISIiCpsOH1TEILSACILQqVdRDsZ3SEREFIgOG1TUajUAwGazBeV83n4q4VpFOZJ4v0Pvd0pERBQuHXZ4slKpRFxcHEpLSwEABoMBgiAEfD6FxwWx1gmbDbCrOkcLgyiKsNlsKC0tRVxcHJRKpdwlERFRJ9NhgwoApKSkAIAvrLRHlaMWFTYXKtUKVMdo232+aBIXF+f7LomIiMKpQwcVQRCQmpqKpKQkuFyudp1rx4nzWPTZPnTvYsQbs7ODVGHkU6vVbEkhIiLZdOig4qVUKtv9xza9axxOV7pR7qiGVqtt120kIiIiap0O25k22C6J00MhADanG2erHHKXQ0RE1CkwqLSSRqVAqlkPACgsC85IIiIiImoZg0obZCQYAAD55xlUiIiIwoFBpQ0yE6WgUsAWFSIiorBgUGmDDG9QYYsKERFRWDCotIH31g9bVIiIiMKDQaUNMhOMAIB8BhUiIqKwYFBpA2+LytlKB2zOWpmrISIi6vgYVNrAbFDDrJcW5issq5G5GiIioo6PQaWN6ocoV8tcCRERUcfHoNJGGRyiTEREFDYMKm2UyZE/REREYcOg0kYcokxERBQ+DCptxEnfiIiIwkfWoOJ2u7Fw4UJ0794der0ePXv2xOLFiyGKopxltcjbolJYboPbE7l1EhERdQQqOS/+/PPPY/ny5Vi1ahX69++PXbt24c4774TZbMaDDz4oZ2kXlGrWQ60U4HKLKLHacUmcXu6SiIiIOixZg8o333yD66+/HlOnTgUAZGVl4e2338aOHTvkLKtFSoWAbvEGnDhXjfzz1QwqREREISTrrZ/LL78cmzdvxpEjRwAAP/zwA7Zu3YrJkyc3e7zD4YDVavXb5OC7/cMOtURERCEla4vKo48+CqvViuzsbCiVSrjdbixZsgQ5OTnNHp+Xl4ennnoqzFU2lZnonfSNQYWIiCiUZG1Ree+997B69WqsWbMGu3fvxqpVq/Diiy9i1apVzR7/2GOPwWKx+LbCwsIwVyzhEGUiIqLwkLVF5eGHH8ajjz6KW265BQAwcOBA5OfnIy8vD7Nnz25yvFarhVarDXeZTTCoEBERhYesLSo2mw0KhX8JSqUSHo9Hpopah9PoExERhYesLSrTpk3DkiVLkJGRgf79+2PPnj14+eWXcdddd8lZ1kV5W1QqbC5Yaly+FZWJiIgouGQNKq+99hoWLlyI+++/H6WlpUhLS8Nvf/tbPPHEE3KWdVEGjQpdYrQ4V+VAwXkbBnYzy10SERFRhyRrUDGZTFi2bBmWLVsmZxkByUw0SEGljEGFiIgoVLjWT4C8qyjnl1XLXAkREVHHxaASoHRO+kZERBRyDCoB4qRvREREocegEiDOpUJERBR6DCoB8s6lUlRRA2dtZM/7QkREFK0YVALUNUYLvVoJjwicrqiRuxwiIqIOiUElQIIg8PYPERFRiDGotIN35E/BeQ5RJiIiCgUGlXbI5Jo/REREIcWg0g4cokxERBRaDCrtkM4+KkRERCHFoNIOmQ2CiiiKMldDRETU8TCotMMl8XoIAmBzunGuyil3OURERB0Og0o7aFVKpJn1AHj7h4iIKBQYVNopPcEbVDhEmYiIKNgYVNopM8EIACg4z9lpiYiIgo1BpZ28a/7ks0WFiIgo6BhU2sk7jX4h+6gQEREFHYNKO3HSNyIiotBhUGknb4tKaaUDNU63zNUQERF1LAwq7RRn0CBWpwIAFJazVYWIiCiYGFSCIIO3f4iIiEKCQSUIfEOU2aGWiIgoqBhUgsDbolJwnkOUiYiIgolBJQgyuIoyERFRSDCoBIF3FeV8BhUiIqKgYlAJgvS6oHKqrAZujyhzNURERB0Hg0oQpMXpoVIIcLo9OGO1y10OERFRh8GgEgRKhYBu8dIqyhyiTEREFDwMKkGSkSgNUeaaP0RERMHDoBIk9R1qOUSZiIgoWBhUgqR+iHKNzJUQERF1HAwqQcJJ34iIiIKPQSVIMjiXChERUdAxqASJN6hU2Fyw1LhkroaIiKhjYFAJEqNWhS4xGgAc+UNERBQssgaVrKwsCILQZMvNzZWzrIBxzR8iIqLgkjWo7Ny5E8XFxb5t48aNAICbb75ZzrICllk3lwonfSMiIgoOlZwX79q1q9/r5557Dj179sRVV10lU0Xtk84WFSIioqCSNag05HQ68c9//hMPPfQQBEFo9hiHwwGHw+F7bbVaw1Veq2T6ggqHKBMREQVDxHSm/eijj1BRUYE5c+Zc8Ji8vDyYzWbflp6eHr4CW8E7lwpv/RAREQVHxASVv/3tb5g8eTLS0tIueMxjjz0Gi8Xi2woLC8NY4cV5W1SKKmrgcntkroaIiCj6RcStn/z8fGzatAkffvhhi8dptVpotdowVdV2XU1a6NQK2F0enC6vQVYXo9wlERERRbWIaFFZuXIlkpKSMHXqVLlLaRdBEDhEmYiIKIhkDyoejwcrV67E7NmzoVJFRANPu2Qk1A1RZlAhIiJqN9mDyqZNm1BQUIC77rpL7lKCwtuiwtlpiYiI2k/2JowJEyZAFEW5ywiaTN/IHw5RJiIiai/ZW1Q6Gt8qyhyiTERE1G4MKkHmnUulsMzWoVqKiIiI5MCgEmTd4vUQBKDa6cb5aqfc5RAREUU1BpUg06qUSI3VAeAQZSIiovZiUAkB7+2fAvZTISIiahcGlRDgpG9ERETBwaASApmJdZO+sUWFiIioXRhUQiDd16LCuVSIiIjag0ElBDJ564eIiCgoGFRCwNtH5YzVAbvLLXM1RERE0YtBJQTiDGqYdNLqBFzzh4iIKHAMKiEgCEKDNX8YVIiIiALFoBIiHKJMRETUfgwqIZKRIA1RZlAhIiIKHINKiNSvoswhykRERIFiUAkRbx8VtqgQEREFjkElRLwtKoXlNfB4RJmrISIiik4MKiGSatZBpRDgrPXgTKVd7nKIiIiiEoNKiKiUCnSL1wPgEGUiIqJAMaiEUDqHKBMREbULg0oI+TrUskWFiIgoIAwqIeQboswWFSIiooAwqIQQJ30jIiJqHwaVEPJNo89J34iIiALCoBJCGXV9VMptLljtLpmrISIiij4MKiEUo1WhS4wGADvUEhERBYJBJcS8Q5QL2U+FiIiozRhUQiyTI3+IiIgCxqASYvWrKDOoEBERtRWDSohlJEpDlHnrh4iIqO0YVEKsftI3DlEmIiJqKwaVEPNOo19UYYfL7ZG5GiIioujCoBJiSSYttCoF3B4RRRU1cpdDREQUVRhUQkwQhPoZatlPhYiIqE0YVMLAe/uHI3+IiIjahkElDNLZokJERBQQBpUwyPQtTsigQkRE1BayB5XTp0/jtttuQ2JiIvR6PQYOHIhdu3bJXVZQeRcn5Oy0REREbaOS8+Ll5eUYPXo0rr76aqxfvx5du3bF0aNHER8fL2dZQZeRUD/pmyiKEARB5oqIiIiig6xB5fnnn0d6ejpWrlzp29e9e3cZKwqNbvF6CAJQ5ahFWbUTiTFauUsiIiKKCrLe+lm3bh2GDRuGm2++GUlJSbjsssvw17/+9YLHOxwOWK1Wvy0a6NRKpMTqALBDLRERUVvIGlR+/vlnLF++HL1798aGDRtw33334cEHH8SqVauaPT4vLw9ms9m3paenh7niwHEuFSIiorYTRFEU5bq4RqPBsGHD8M033/j2Pfjgg9i5cye+/fbbJsc7HA44HA7fa6vVivT0dFgsFsTGxoal5kA9/P4PeP/7U3jo2kvx4LjecpdDREQkG6vVCrPZ3Kq/37K2qKSmpqJfv35++/r27YuCgoJmj9dqtYiNjfXbooV30je2qBAREbWerEFl9OjROHz4sN++I0eOIDMzU6aKQiedc6kQERG1WUBBpbCwEKdOnfK93rFjB+bPn4/XX3+9Tef53e9+h+3bt+PZZ5/FsWPHsGbNGrz++uvIzc0NpKyIlpkoDVFmiwoREVHrBRRUbr31Vnz55ZcAgJKSElx77bXYsWMHHn/8cTz99NOtPs/w4cOxdu1avP322xgwYAAWL16MZcuWIScnJ5CyIpp3dtoSqx12l1vmaoiIiKJDQEFl//79GDFiBADgvffew4ABA/DNN99g9erVePPNN9t0ruuuuw779u2D3W7HwYMH8Zvf/CaQkiJenEENk1aatuZUOVtViIiIWiOgoOJyuaDVSpOWbdq0Cb/61a8AANnZ2SguLg5edR2IIAj1U+mznwoREVGrBBRU+vfvj7/85S/4+uuvsXHjRkyaNAkAUFRUhMTExKAW2JF451JhUCEiImqdgILK888/jxUrVmDs2LGYNWsWBg8eDECaadZ7S4iayuAQZSIiojYJaK2fsWPH4ty5c7BarX4LCN5zzz0wGAxBK66j4ey0REREbRNQi0pNTQ0cDocvpOTn52PZsmU4fPgwkpKSglpgR5KZwCHKREREbRFQULn++uvxj3/8AwBQUVGBkSNH4qWXXsL06dOxfPnyoBbYkTScndbjkW3lAiIioqgRUFDZvXs3rrjiCgDABx98gOTkZOTn5+Mf//gHXn311aAW2JGkmnVQKQQ4az0orXRc/ANERESdXEBBxWazwWQyAQC++OIL3HDDDVAoFPjlL3+J/Pz8oBbYkaiUClwSrwcA5J+vlrkaIiKiyBdQUOnVqxc++ugjFBYWYsOGDZgwYQIAoLS0NKoWCpQDO9QSERG1XkBB5YknnsCCBQuQlZWFESNGYNSoUQCk1pXLLrssqAV2NAwqRERErRfQ8OSbbroJY8aMQXFxsW8OFQAYN24cZsyYEbTiOiJO+kZERNR6AQUVAEhJSUFKSopvFeVu3bpxsrdWyOSkb0RERK0W0K0fj8eDp59+GmazGZmZmcjMzERcXBwWL14Mj8cT7Bo7lAzOpUJERNRqAbWoPP744/jb3/6G5557DqNHjwYAbN26FYsWLYLdbseSJUuCWmRH4p1Gv6zaiUq7CyadWuaKiIiIIldAQWXVqlV44403fKsmA8CgQYNwySWX4P7772dQaUGMVoVEowbnq50oKLOhf5pZ7pKIiIgiVkC3fsrKypCdnd1kf3Z2NsrKytpdVEeXXtehtpC3f4iIiFoUUFAZPHgw/vjHPzbZ/8c//hGDBg1qd1EdnbdDLUf+EBERtSygWz9Lly7F1KlTsWnTJt8cKt9++y0KCwvx73//O6gFdkS+IcpsUSEiImpRQC0qV111FY4cOYIZM2agoqICFRUVuOGGG3DgwAG89dZbwa6xw8ngrR8iIqJWEURRDNoyvj/88AOGDBkCt9sdrFO2yGq1wmw2w2KxRNXU/TtOlGHmim+RkWDAf39/tdzlEBERhVVb/n4H1KJC7eNtUTldUYNaN+edISIiuhAGFRkkmbTQqhRwe0QUVdjlLoeIiChiMajIQKEQfEOUOUMtERHRhbVp1M8NN9zQ4vsVFRXtqaVTyUww4FhpFfLLqjEGXeQuh4iIKCK1KaiYzS3Pomo2m3HHHXe0q6DOwteiwrlUiIiILqhNQWXlypWhqqPT4SrKREREF8c+KjLh7LREREQXx6Aik4aTvgVxKhsiIqIOhUFFJt3ipaBS6ahFuc0lczVERESRiUFFJjq1EimxOgDsp0JERHQhDCoyyvD1U6mWuRIiIqLIxKAiowwOUSYiImoRg4qMMjk7LRERUYsYVGTku/XDoEJERNQsBhUZNRyiTERERE0xqMgoM9EIACix2mF3uWWuhoiIKPLIGlQWLVoEQRD8tuzsbDlLCqt4gxoxWhVEEThVXiN3OURERBGnTWv9hEL//v2xadMm32uVSvaSwkYQBGQkGPBTsRUFZdXolRQjd0lEREQRRfZUoFKpkJKSIncZsvEGFa75Q0RE1JTsfVSOHj2KtLQ09OjRAzk5OSgoKLjgsQ6HA1ar1W+LdlxFmYiI6MJkDSojR47Em2++ic8//xzLly/HiRMncMUVV6CysrLZ4/Py8mA2m31benp6mCsOPu8QZU76RkRE1JQgRtDSvRUVFcjMzMTLL7+Mu+++u8n7DocDDofD99pqtSI9PR0WiwWxsbHhLDVovj56Frf/bQd6J8Vg40NXyV0OERFRyFmtVpjN5lb9/Za9j0pDcXFxuPTSS3Hs2LFm39dqtdBqtWGuKrQyE6QhygVlNng8IhQKQeaKiIiIIofsfVQaqqqqwvHjx5Gamip3KWGTGqeDUiHAUevB2SrHxT9ARETUicgaVBYsWIAtW7bg5MmT+OabbzBjxgwolUrMmjVLzrLCSq1U4JI4PQBw5A8REVEjsgaVU6dOYdasWejTpw9mzpyJxMREbN++HV27dpWzrLDzTqWff75a5kqIiIgii6x9VN555x05Lx8xMhINwDGu+UNERNRYRPVR6awyE7iKMhERUXMYVCKA99YPJ30jIiLyx6ASATjpGxERUfMYVCKAt0XlfLUTVY5amashIiKKHAwqEcCkUyPBqAHAVhUiIqKGGFQiRLqvnwqHKBMREXkxqESITHaoJSIiaoJBJUJkJnonfWNQISIi8mJQiRDpbFEhIiJqgkGlOR43sHcNULgzbJfkrR8iIqKmGFSas2Up8NF9wBePA6IYlkt651I5XV6DWrcnLNckIiKKdAwqzRk6B1DpgcLvgMP/Dsslk006aFQK1HpEFFvsYbkmERFRpGNQaU5sKvDL+6Tnm5+WbgWFmEIhID1eD4AdaomIiLwYVC5k9DxAFwecPQT88HZYLpmZaATAfipEREReDCoXoo8DrlwgPf/yWcBVE/JLZvhWUeakb0RERACDSsuG/waI7QZYTwM7/hryy3mDSiFbVIiIiAAwqLRMrQOufkx6/vVLQE1FSC/HSd+IiIj8MahczOBZQNe+gL0C2LYspJfytqgUnLdBDNOwaCIiokjGoHIxCiUw7gnp+fblgLUoZJfyzk5b6ahFhc0VsusQERFFCwaV1ugzGUj/JVBrB756LmSX0amVSI7VAgDy2U+FiIiIQaVVBAG49inp+Z63gLNHQnapzAQOUSYiIvJiUGmtjF8CfaYAogf4z9Mhu4xvccLzHKJMRETEoNIW454ABAVw8JOQLVjoHfnDFhUiIiIGlbZJ6gsMvlV6vmlRSBYs5BBlIiKiegwqbXX1Y4BSC+RvBY5tCvrp0znpGxERkQ+DSluZuwEjfiM937QI8HiCevrMuqBSbLXDURv6xRCJiIgiGYNKIK74H0BrBs7sB/a9H9RTJxg1MGqUEEWgsCz06wsRERFFMgaVQBgSgDHzpOf/eQaodQTt1IIgIKNuFWXe/iEios6OQSVQI+8DYlIASwGw6+9BPXVGgh4AkM8hykRE1MkxqARKYwDGPio9/+8LgN0atFNnJnonfeOtHyIi6twYVNrjstuBxF6A7TzwzWtBO61vccIytqgQEVHnxqDSHkpV/YKF3/4JqDwTlNPWBxX2USEios6NQaW9+v4KuGQo4KoG/rs0KKdsODutGIJJ5YiIiKIFg0p7CQIwvm7Bwu/fBM4fb/cp0+L0UCoE2F0elFYGb0QRERFRtGFQCYbuVwC9rgU8tdJw5XZSKxVIi9MB4O0fIiLq3BhUgmX8kwAE4MCHQNGedp/O20+Fa/4QEVFnxqASLCkDgYE3S883LWr36TISvEOUGVSIiKjzipig8txzz0EQBMyfP1/uUgJ3zeOAQg38/BVw/Mt2ncrXoZaTvhERUScWEUFl586dWLFiBQYNGiR3Ke0TnwUMv1t63s4FCzlEmYiIKAKCSlVVFXJycvDXv/4V8fHxcpfTflc+DGhMQPFe4Ke1AZ+GQYWIiCgCgkpubi6mTp2K8ePHX/RYh8MBq9Xqt0UcYxfg8gek55sXA25XQKfJqLv1c67KiSpHbbCqIyIiiiqyBpV33nkHu3fvRl5eXquOz8vLg9ls9m3p6ekhrjBAo3IBY1eg/IQ0t0oAYnVqxBvUALiKMhERdV6yBZXCwkLMmzcPq1evhk6na9VnHnvsMVgsFt9WWFgY4ioDpI0BrnpEer5lKeCoCug0HKJMRESdnWxB5fvvv0dpaSmGDBkClUoFlUqFLVu24NVXX4VKpYLb7W7yGa1Wi9jYWL8tYg2ZDcR3B6pLge1/DugUGXWrKLNFhYiIOivZgsq4ceOwb98+7N2717cNGzYMOTk52Lt3L5RKpVylBYdKA1zzf9Lzba8C1efafIpMb4sKV1EmIqJOSiXXhU0mEwYMGOC3z2g0IjExscn+qNX/BmDbH4CSH4H/vghMfq5NH68f+VMTiuqIiIginuyjfjo0hQK4tm7Bwl1/A8rz2/TxDE76RkREnZxsLSrN+eqrr+QuIfh6XgN0vwo4sQX48lnghhWt/qi3ReVUeQ1q3R6olMyVRETUufAvXziMXyQ9/vguULK/1R9LidVBo1Sg1iOi2GIPTW1EREQRjEElHC4ZAvSfAUAENj/V6o8pFAK6JegBcIZaIiLqnBhUwuWahYBCBRz9Aji5tdUf8478efObk6hxNh2yTURE1JExqIRLYk9pbhUA2PgkIIqt+tgdo7KgUgjY+NMZ3LziGxRVcAQQERF1Hgwq4XTVI4DaAJzeBRz8pFUfuTo7Cat/PRIJRg32n7biV3/cil0ny0JcKBERUWRgUAknU7K0DhAAbH4acLduscGRPRKxbu5o9E2NxbkqJ2b9dTve3VkQwkKJiIgiA4NKuF3+IKBPAM4fBfaubvXHusUb8K/7RmHKwBS43CIe+dc+LFp3ALVuTwiLJSIikheDSrjpYoErH5aef5UHOFs/msegUeFPtw7BQ9deCkDqYDt75Q6UVztDUSkREZHsGFTkMPxuwJwBVBYDO1o/ARwACIKAB8f1xl9uGwqDRoltx85j+p+34ciZyhAVS0REJB8GFTmotMA1j0vPv34FsLW9c+ykASn48P7L0S1ej/zzNsz40zZs+ulMkAslIiKSF4OKXAbeDCT1BxwWYOsrAZ0iOyUW6+aOwS97JKDa6cZv3tqFP315DGIrhz4TERFFOgYVuSiU9VPrf7cCsJwK6DQJRg3eunskbv9lJkQReGHDYTzw9h5ODkdERB0Cg4qcel8LZI4G3A6pY22A1EoFFk8fgGdnDIRKIeDTH4s5ORwREXUIDCpyEgRgfN3aP3vXAKWH2nW6W0dmNJkc7vt8Tg5HRETRi0FFbunDgezrANEjTQLXTo0nh7vl9e14b2dhEAolIiIKPwaVSDDuSUBQAIc/Awq+a/fpvJPDTR4gTQ73+3/9yMnhiIgoKjGoRIKulwKX3SY939T6BQtb4p0c7nfj6yeHm7NyJypsnByOiIiiB4NKpBj7GKDSAQXfAkc2BOWUCoWAeePrJ4fbeuwcrv8TJ4cjIqLowaASKWLTgJH3Ss83LQI8wRtezMnhiIgoWjGoRJIx8wGdGTh7EPjx3aCempPDERFRNGJQiST6eOCK/5Ge/2cJ4LIH9fTNTQ734Dt7OTkcERFFLAaVSDPiHiD2EsB6Ctj5RtBP750cbsmMAVApBHzyQxEnhyMioojFoBJp1HqpYy0AfP0iYLeE5DI5IzMbTQ63jZPDERFRxGFQiUSDZwFd+gA15cC2P4TsMiN7JOLj3NHITjHhXJWDk8MREVHEYVCJREoVMP5J6fm3fwYqS0J2qfQEAz68/3JODkdERBGJQSVS9ZkCpI8EamuAr54L6aU4ORwREUUqBpVIJQjA+EXS893/kFpWLKdCdrkLTQ53lJPDERGRjAQxiifSsFqtMJvNsFgsiI2Nlbuc0Hj7VmkNIK+0y4C+04DsadLU+yFwsNiK3/xjF06V1yBGq8Ky//cLjO+XHJJrERFR59OWv98MKpHOaQO+fxM4+Ik0vT4a/OPq0gfoe50UXFJ/IbXCBElZtRP3/fN7fHeiDIIALJjQB/eP7QkhiNcgIqLOiUGlo6oqBQ7/Gzj4KfDzV4DHVf+eOR3Ivk4KLhmjAIWy3ZdzuT14+pOf8Nb2fADAtMFpWHrjIOg17T83ERF1XgwqnYHdAhz5Ajj0CXB0I+Cy1b9n6AL0mQz0/RXQ4ypApW3XpVZ/l48nPz6AWo+IHl2MuGHIJZjYPwW9kmLYwkJERG3GoNLZuGqA419Kt4eOrJfmX/HSmIBLJ0itLb2vBbSmgC7x3c/ncd/q3Sirrh8J1KOrERP7p2Bi/xQM7mZmaCEiolZhUOnM3C4gf5t0e+jQp0Blcf17Si3Q82qpT8ulkwFjYptOXWFzYsOBEmw4cAZbj56Ds8FcK6lmHSb0S8bEASkYkZUAlZIDyoiIqHkMKiTxeICi3cDBdVJrS9nP9e8JCiBzdN0IoqmAuVubTl1pd+Grw2fx+YESfHWoFNUNFjaMN6gxvm8yJvZPwZjeXaBTs08LERHVY1ChpkQRKD0otbIcXAeU7PN/P22IFFr6TgO69G7Tqe0uN7YdO4cNB0qw8aczKLfVd/I1apQY2ycJE/on45rsJJh06mD8NEREFMUYVOjiyk/W3x4q2A6/Yc9ds+tGEE0DUge3adhzrduDnSfL624RlaDYYve9p1EqcHmvREzqn4Lx/ZLRJaZ9nXyJiCg6RU1QWb58OZYvX46TJ08CAPr3748nnngCkydPbtXnGVSCpPKMNKncwU+BE1sAT239e+YMachz9nVAxi/bNOxZFEXsO23B5/tL8PmBEvx8ttr3nkIAhmUmYOKAFEzsn4xu8YZg/kRERBTBoiaofPLJJ1AqlejduzdEUcSqVavwwgsvYM+ePejfv/9FP8+gEgI1FcDRL6TbQ8c2Nx323HsCkD4C6DYcSOrbpuByrLQSGw6cwef7S7DvtMXvvQGXxGJivxRMGsBhz0REHV3UBJXmJCQk4IUXXsDdd9990WMZVELMaQOO/0e6PXT439LcLQ1pYoBLhkihxbsZu7Tq1KfKbfjiwBlsOFCCnSfL4Gnwb2GPLsa6lhYOeyYi6oiiMqi43W68//77mD17Nvbs2YN+/fo1OcbhcMDhcPheW61WpKenM6iEg9sFnNwqbad2Aqd3A85mFiyM714fWtKHA8kDAGXLHWjPVzmw6aDU0rLt2Pnmhz33T8GI7hz2TETUEURVUNm3bx9GjRoFu92OmJgYrFmzBlOmTGn22EWLFuGpp55qsp9BRQYeN3D2kBRaTu0ETu2SXjem0kkLKXYbBnSru2UUm3rB01baXfjy8FlsOFCCLw+VwtZo2PO4vsmYxGHPRERRLaqCitPpREFBASwWCz744AO88cYb2LJlC1tUolFNBXD6eym0nNohBZjGt4sAILab1NribXlJHdzsNP/eYc+f7y/BpoP+w54NGiV+2SMRw7LiMTwrAQMvMTO4EBFFiagKKo2NHz8ePXv2xIoVKy56LPuoRDiPByg7DhTuqG91KT0AiB7/45QaIGVQXXAZJnXWNaf7DYtuadgzIA19HtjNLAWXzAQMzYxHvFETjp+SiIjaKKqDyjXXXIOMjAy8+eabFz2WQSUKOaqk2XK9waVwB2A71/S4mGT/TrppvwA0RgDSsOcDRVZ8d6IMu06WYefJcpyrcjQ5Re+kGAzLSsDwrHgMy0xAeoKeHXOJiCJA1ASVxx57DJMnT0ZGRgYqKyuxZs0aPP/889iwYQOuvfbai36eQaUDEEVp8rmGt4tK9vnP5QIAghJI7l8/NLrbcKnjrkIBURSRf96GXfnldcGlDMcbzNnilWTSYnhWgu92UXaKiZ1ziYhkEDVB5e6778bmzZtRXFwMs9mMQYMG4ZFHHmlVSAEYVDosVw1Q/EODW0Y7/RdX9FKoAfMl0m2iuAzp0dwNiEtHuToZu8qN2HmqGjtPlmH/aQtcbv9/1Y0aJYZkSq0tw7Pi8YuMOBg0qjD9kEREnVfUBJX2YlDpRCyn/G8XFe8F3M6Lfy4mGTCnwx17Cc4ISTjmjMduSwy+LtXhqCMeVhh9hyoVAgakxWJoXXAZmhWPJJMudD8TEVEnxaBCHZ+7FqgskgJMRSFgKWjwvFB6rK256GmcqhicU3TFidoEHHcmoEhMxGmxi28zJKRhaPcuUj+XrAT06GJkPxcionZiUCESRcBWJgWYikIpxFgKgYqC+ue28xc9jVNUorguvBShC8pUydAmZqJLt57I6pmN3r2zodFxnSIiorZoy99v3pCnjkkQAGOitKVd1vwxzmrAcrpBmCn0a5URrUXQwI1MoRSZKJU+IwI4V7ftlXZVKOJRo0uCaEqFPrEbYrtmQGlOA0yp0uR2plRAH9+mVaiJiEjCoEKdl8YIdL1U2pohuGulTrx1Aaa2LB/lxcdhP3sSysrTiHedgR4OxHnKEWcrB2yHgTPNX0pUaiGYUoDYNMCUApjqHn2v6wKNhq0zREQNMagQXYhSBcSlSxuk/1i6Nnhb9Hhw4tQpHDt6EGeL8lF1tgBuSzHi3eeQLJQjRShHklCORKESgtsBVORLW0t05guHGG/rjDFJqo2IqBPgbzuiAAkKBbpnZKB7RoZvnyiKOFVegwNFFqwvsuKnYiuOnj4Pd2UJklHmF2BShDJ0U1bgEpUFiZ7z0Hjs0pIDdgtw9mBLF5bCSmxqg1BTF2JiUgBDPGBIlDZNDG85EVFUY2daojAoq3biYLEVPxVZcaDIgp+KrTh+thpuj/c/PxEm1CBZKEO6yoLBcTXoa6xGlsaCZKEcJtdZKKvOAJUlgOhu8Vp+FOq60JIgPerj/V8bEgG993nde9pYhhsiCimO+iGKAnaXG0fOVOJAkRRgfiq24mCx1W/FaC+FAHTvYkT/1BgMTazFAHMNeuusiHWdk/rRVBYD1mKguhSwlUvLEtTam7lqKyhUDcJLgrT5vU5s+r7WDCg4yy8RtQ6DClGUcntE5J+vxk/FVr8Ac7ay6VpGgLQsQL+0WPRLjUX/NDOyU03ISDBArVQAThtQUyYNw7bVPdaU171uuK+s7nkZ4Gq69ECrCMqmrTXaWKnDsjZGetTE1G11z5vbrzYw8BB1AgwqRB1MaaXdF1p+qgswJ85Xo7n/etVKAZmJRvTsakTPrjHSlhSDHl2NiNWpW76Qy94g3DQXcBqGm7rXzqrg/rC+MOPdTPXPtY3Cjvd5S2FIpeOtLKIIw6BC1AlUO2pxqKSyLrxYcKDIiqNnqlDjunAfliSTFr2S6sJLVyN61j1PNesCn3G31iEFlsatN45Kaa4aZ1XdVi2tnt14n7NaOhYh+lUkKAClFlBpAKWmwfOGj1rpPe+jUnPh91Raab9S3XRfs+dV+z9XqKQWKIWKrUfUaTGoEHVSHo+IYqsdx0urcPxs3VZajeNnq1B6gdtHAGDQKNGjYQtM1xj0TDIiK9EInVoZ+sJFUVqM0lkNOCsbBJiGgeZigaeqwf7qwG9jhZsvtNQ9Cor61w0DTeOAE9DnlA2OU/l/vsV9F3vt3acM8LzqBvWx9aszYFAhoiasdlddgKmuCzBSkMk/b0Otp/lfAwoBSE8w1LfA1N1G6tk1BglGTZh/gjbyuAGXTQottQ5pEctaB+B2ALVO6dHtarrP91j3Gbez6b7Gj83ta3xe0SP3NxIdvKHF2/rU8Llvn1qaS6jZ5+q6AKRu5hx17/meq+o+2+BYlU5q/fI9Nniu1Db/niIMYb6DYVAholZzuT0oKLP5QsyxBq0xlfbaC34u3qD2tb70SpJaYHp2jUG3eAOUCv5fcRMetxRgPG7AUysFF09tg9fuuufuuue1jV43PK4W8Hgafe5C5/Q0+lzjazQ43m9rvC8Yrxvsa8sw+0jnDThKTaOg0+i17/2GQUfb/DGCov6freiu+2frabCv8XOP/78rjff5ve9p9PmG73uanr/7lcCVC4L6lXGtHyJqNbVS4QscDYmiiLNVDt+tI2mrxvHSKpyuqEG5zYVd+eXYlV/u9zmNUoHuXYzI6mJARkLdlmhERoIBl8TpoVF10n4ZCiWg0MtdReQQxbrg4pKCi9tVH2Lcjfe5pBXTfc9d9Z9t8rm697zPWzy20ef8Wtgc0hD/WkeDzV7/vqdBiPfUBr9TeSQxdpH18gwqRNQsQRCQZNIhyaTDqJ6Jfu/ZnLU4cc7b+lJ/K+nnc9Vw1npw+EwlDp+pbHJOhQCkmvUNAkx9mMlMNMCsVwfeqZeiiyBIt12idTkId22DQNMw1Njrbv3Zmwk79lYEobp9EKVWFUFZ37dIUDTog9TMPr/3FY0+732/8Tlbca64jIt+HaHEWz9EFDRuj4iiihocO1uFgvM25J+3oaDMhsIy6bGlEUkAYNKpfKElPaFBiEkwIjVOJ80PQ0RRj7d+iEgWSoWA9AQpZDTmvZXkDS0NQ0z+eRtKKx2otNfiQJE02V1z506L0yEzwegLMZl1LTLpCVJrDBF1PAwqRBQWDW8lDc1MaPJ+jdONU+XNhJi6R0etB4VlNSgsq2n2/HEGtS+0SK0w9SEmOVbXefvGEEU5BhUiigh6jRK9k03onWxq8p7HI7XGNG2JqUZBWQ3OVTlQYXOhwmbBj6cszZ6/S4wGSSYdUsw6JMfqkByrRUqsDslmnfQYq0O8gX1kiCIN+6gQUdSzOWtRWFZTF1zq+8Tkl9lwqqwGTnfr5jDRqBS+AJMUKwUY6bm0zxtywjIJHlEHxj4qRNSpGDQq9EkxoU9K09YYURRRbnOhxGLHGau0lXgfLXacsTpwxmrH+WonnBe5veRl1qt9rTHJJq0vwHhbZpLNWnQxaqHgfDJE7cagQkQdmiAISDBqkGDUoF/ahf/PzVHrRqnVgdJKO0osDl+YqQ80UsCxuzyw1LhgqXE1OwTbS6UQkGTS1rfMmHXoEqOBSaeGSaeCSadGjFYFk06F2Lp9MToVRzYRNcKgQkQEQKtSXnDEkpcoirDaa33hpcRqR2ldgCmxOHzB5myVA7UeEUUWO4os9jbVoVMrpDBTF2K8wUYKNd6Q0/x7sXVhR69Wsq8NdRgMKkRErSQIAsx6Ncx6NS5tptOvV63bg7NVDr9bSyVWO85XOVDlqEWlvRZWey0q7S5U2aXX3jlm7C4P7C4HzrawiOTFqBQCYurCTIxWXddq4x92YvVqxOrUiNWr6h6loOPdz1FSFCkYVIiIgkylVCDVrEequfVT5rvcHlT7Qkx9gKl0SM+t3td2ly/sVNpddY/1+z0iUOsR60ZBuQC03N/mQnRqhS/AmBuFGP9w03zY4S0sChYGFSKiCKBWKhBn0CDOEPiq1KIowuZ014eYBoGmqkGg8YYeq90Fa430Wnp0+Rai9LbslAbYsmPQKFtssWm4P96gQbxRjYS6n5+tOdQQgwoRUQchCAKMWhWMWhVSzLqAzuH2iKhySMHFUhderDXNhxq//XXvVTmkoGNzumFzulHSdJLhizJpVYirCy7xRo0UZAwaJBjVjV5rEG9QM9x0cAwqRETko1TU98NJD+DztW5PXdBpGG7qQ42lxj/wWGpcKLc5UWGTHj0ipJYgR+1Fh4k3ZNKq6kKMFGa8rTPecFP/Wmq9iTdoeHsqSjCoEBFR0KjacQvL4xFhtbtQVu1Euc2F8monymxOlDd6XWFz+o6paBRuCspafz1fuKkLOAkGDWL1asQZpKAWZ1AjTl+/L06vZv8bGTCoEBFRRFAohDaHHP9w40RZtdQy4ws11S5f2Clr0HIj+oUbW5vqjNGqfK1ODUONWa/x36dXw+x7XwOjhsPGA8GgQkREUSuQcOP2iLDW3XLyhZu6oGOpcaGixgWLzVX3vG6frb6jcZVD6otzuqJtI6pU3ttqDYJMnEHjCz2+VhyDusmoqs48Nw6DChERdSpKheC75dMWtW4PKu21UpCpkW47WXzPGz7Whxtv6HG6Paj1iDhf7cT5ameba1YphGbnv/HObNxwVJV38r/YultVJp0KMRpV1C7pwKBCRETUCiqlIqCAI4qib+mFihqnL9Q0bLXx7asLOA1HWbk9Imo90ppV5TZXQLULgtQnxxt0Lhh6Gg8f16kRZ5Qe5cKgQkREFEKCIECvUUKvUbZ52LgoiqhxuZuMoqr0DRNvMFy8wb7KBqOtnG4PRBHS+/ZatHUSwEn9U/CX24e26TPBxKBCREQUoQRBgEGjgkET+Nw4dpe7ybw39RP+NbfPPwDF6uWNCgwqREREHZhOrYROrUTShZenapHHIwa3oDaSdTB4Xl4ehg8fDpPJhKSkJEyfPh2HDx+WsyQiIiJqQO5OuLIGlS1btiA3Nxfbt2/Hxo0b4XK5MGHCBFRXV8tZFhEREUUIQRRFedt0Gjh79iySkpKwZcsWXHnllRc93mq1wmw2w2KxIDY2NgwVEhERUXu15e93RPVRsVgsAICEhIRm33c4HHA46lfytFoDWO2KiIiIokbELFjg8Xgwf/58jB49GgMGDGj2mLy8PJjNZt+Wnh7IkllEREQULSLm1s99992H9evXY+vWrejWrVuzxzTXopKens5bP0RERFEk6m79zJ07F59++in++9//XjCkAIBWq4VWqw1jZURERCQnWYOKKIp44IEHsHbtWnz11Vfo3r27nOUQERFRhJE1qOTm5mLNmjX4+OOPYTKZUFJSAgAwm83Q6/VylkZEREQRQNY+KhdasnrlypWYM2fORT/P4clERETRJ2r6qERIP14iIiKKUBEzPJmIiIioMQYVIiIiilgMKkRERBSxImIelUB5+7hwKn0iIqLo4f273Zq+qlEdVCorKwGAU+kTERFFocrKSpjN5haPiZgp9APh8XhQVFQEk8l0waHOgfJOz19YWMihz3X4nTTF76R5/F6a4nfSPH4vTXWG70QURVRWViItLQ0KRcu9UKK6RUWhULQ45X4wxMbGdth/UQLF76QpfifN4/fSFL+T5vF7aaqjfycXa0nxYmdaIiIiilgMKkRERBSxGFQuQKvV4sknn+RqzQ3wO2mK30nz+L00xe+kefxemuJ34i+qO9MSERFRx8YWFSIiIopYDCpEREQUsRhUiIiIKGIxqBAREVHEYlBpxp/+9CdkZWVBp9Nh5MiR2LFjh9wlySovLw/Dhw+HyWRCUlISpk+fjsOHD8tdVkR57rnnIAgC5s+fL3cpsjp9+jRuu+02JCYmQq/XY+DAgdi1a5fcZcnK7XZj4cKF6N69O/R6PXr27InFixe3ao2TjuK///0vpk2bhrS0NAiCgI8++sjvfVEU8cQTTyA1NRV6vR7jx4/H0aNH5Sk2jFr6XlwuFx555BEMHDgQRqMRaWlpuOOOO1BUVCRfwTJhUGnk3XffxUMPPYQnn3wSu3fvxuDBgzFx4kSUlpbKXZpstmzZgtzcXGzfvh0bN26Ey+XChAkTUF1dLXdpEWHnzp1YsWIFBg0aJHcpsiovL8fo0aOhVquxfv16/PTTT3jppZcQHx8vd2myev7557F8+XL88Y9/xMGDB/H8889j6dKleO211+QuLWyqq6sxePBg/OlPf2r2/aVLl+LVV1/FX/7yF3z33XcwGo2YOHEi7HZ7mCsNr5a+F5vNht27d2PhwoXYvXs3PvzwQxw+fBi/+tWvZKhUZiL5GTFihJibm+t77Xa7xbS0NDEvL0/GqiJLaWmpCEDcsmWL3KXIrrKyUuzdu7e4ceNG8aqrrhLnzZsnd0myeeSRR8QxY8bIXUbEmTp1qnjXXXf57bvhhhvEnJwcmSqSFwBx7dq1vtcej0dMSUkRX3jhBd++iooKUavVim+//bYMFcqj8ffSnB07dogAxPz8/PAUFSHYotKA0+nE999/j/Hjx/v2KRQKjB8/Ht9++62MlUUWi8UCAEhISJC5Evnl5uZi6tSpfv/OdFbr1q3DsGHDcPPNNyMpKQmXXXYZ/vrXv8pdluwuv/xybN68GUeOHAEA/PDDD9i6dSsmT54sc2WR4cSJEygpKfH7b8hsNmPkyJH8vduIxWKBIAiIi4uTu5SwiupFCYPt3LlzcLvdSE5O9tufnJyMQ4cOyVRVZPF4PJg/fz5Gjx6NAQMGyF2OrN555x3s3r0bO3fulLuUiPDzzz9j+fLleOihh/C///u/2LlzJx588EFoNBrMnj1b7vJk8+ijj8JqtSI7OxtKpRJutxtLlixBTk6O3KVFhJKSEgBo9veu9z0C7HY7HnnkEcyaNatDL1TYHAYVapPc3Fzs378fW7dulbsUWRUWFmLevHnYuHEjdDqd3OVEBI/Hg2HDhuHZZ58FAFx22WXYv38//vKXv3TqoPLee+9h9erVWLNmDfr374+9e/di/vz5SEtL69TfC7Wey+XCzJkzIYoili9fLnc5YcdbPw106dIFSqUSZ86c8dt/5swZpKSkyFRV5Jg7dy4+/fRTfPnll+jWrZvc5cjq+++/R2lpKYYMGQKVSgWVSoUtW7bg1VdfhUqlgtvtlrvEsEtNTUW/fv389vXt2xcFBQUyVRQZHn74YTz66KO45ZZbMHDgQNx+++343e9+h7y8PLlLiwje3638vds8b0jJz8/Hxo0bO11rCsCg4kej0WDo0KHYvHmzb5/H48HmzZsxatQoGSuTlyiKmDt3LtauXYv//Oc/6N69u9wlyW7cuHHYt28f9u7d69uGDRuGnJwc7N27F0qlUu4Sw2706NFNhq0fOXIEmZmZMlUUGWw2GxQK/1+1SqUSHo9HpooiS/fu3ZGSkuL3e9dqteK7777r1L93gfqQcvToUWzatAmJiYlylyQL3vpp5KGHHsLs2bMxbNgwjBgxAsuWLUN1dTXuvPNOuUuTTW5uLtasWYOPP/4YJpPJd9/YbDZDr9fLXJ08TCZTkz46RqMRiYmJnbbvzu9+9ztcfvnlePbZZzFz5kzs2LEDr7/+Ol5//XW5S5PVtGnTsGTJEmRkZKB///7Ys2cPXn75Zdx1111ylxY2VVVVOHbsmO/1iRMnsHfvXiQkJCAjIwPz58/HM888g969e6N79+5YuHAh0tLSMH36dPmKDoOWvpfU1FTcdNNN2L17Nz799FO43W7f796EhARoNBq5yg4/uYcdRaLXXntNzMjIEDUajThixAhx+/btcpckKwDNbitXrpS7tIjS2Ycni6IofvLJJ+KAAQNErVYrZmdni6+//rrcJcnOarWK8+bNEzMyMkSdTif26NFDfPzxx0WHwyF3aWHz5ZdfNvs7ZPbs2aIoSkOUFy5cKCYnJ4tarVYcN26cePjwYXmLDoOWvpcTJ05c8Hfvl19+KXfpYSWIYieaHpGIiIiiCvuoEBERUcRiUCEiIqKIxaBCREREEYtBhYiIiCIWgwoRERFFLAYVIiIiilgMKkRERBSxGFSIqEMRBAEfffSR3GUQUZAwqBBR0MyZMweCIDTZJk2aJHdpRBSluNYPEQXVpEmTsHLlSr99Wq1WpmqIKNqxRYWIgkqr1SIlJcVvi4+PByDdllm+fDkmT54MvV6PHj164IMPPvD7/L59+3DNNddAr9cjMTER99xzD6qqqvyO+fvf/47+/ftDq9UiNTUVc+fO9Xv/3LlzmDFjBgwGA3r37o1169aF9ocmopBhUCGisFq4cCFuvPFG/PDDD8jJycEtt9yCgwcPAgCqq6sxceJExMfHY+fOnXj//fexadMmvyCyfPly5Obm4p577sG+ffuwbt069OrVy+8aTz31FGbOnIkff/wRU6ZMQU5ODsrKysL6cxJRkMi9KiIRdRyzZ88WlUqlaDQa/bYlS5aIoiitxH3vvff6fWbkyJHifffdJ4qiKL7++utifHy8WFVV5Xv/s88+ExUKhVhSUiKKoiimpaWJjz/++AVrACD+3//9n+91VVWVCEBcv3590H5OIgof9lEhoqC6+uqrsXz5cr99CQkJvuejRo3ye2/UqFHYu3cvAODgwYMYPHgwjEaj7/3Ro0fD4/Hg8OHDEAQBRUVFGDduXIs1DBo0yPfcaDQiNjYWpaWlgf5IRCQjBhUiCiqj0djkVkyw6PX6Vh2nVqv9XguCAI/HE4qSiCjE2EeFiMJq+/btTV737dsXANC3b1/88MMPqK6u9r2/bds2KBQK9OnTByaTCVlZWdi8eXNYayYi+bBFhYiCyuFwoKSkxG+fSqVCly5dAADvv/8+hg0bhjFjxmD16tXYsWMH/va3vwEAcnJy8OSTT2L27NlYtGgRzp49iwceeAC33347kpOTAQCLFi3Cvffei6SkJEyePBmVlZXYtm0bHnjggfD+oEQUFgwqRBRUn3/+OVJTU/329enTB4cOHQIgjch55513cP/99yM1NRVvv/02+vXrBwAwGAzYsGED5s2bh+HDh8NgMODGG2/Eyy+/7DvX7NmzYbfb8corr2DBggXo0qULbrrppvD9gEQUVoIoiqLcRRBR5yAIAtauXYvp06fLXQoRRQn2USEiIqKIxaBCREREEYt9VIgobHinmYjaii0qREREFLEYVIiIiChiMagQERFRxGJQISIioojFoEJEREQRi0GFiIiIIhaDChEREUUsBhUiIiKKWAwqREREFLH+P2rUJDqV3s8AAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# [Model inferecnce and Plot]\n",
    "\n",
    "def plot_loss_history(train_loss_history, val_loss_history):\n",
    "    plt.plot(train_loss_history)\n",
    "    plt.plot(val_loss_history)\n",
    "    plt.title('Loss history')\n",
    "    plt.ylabel('Loss')\n",
    "    plt.xlabel('Epoch')\n",
    "    plt.legend(['train', 'val'], loc='upper left')\n",
    "    plt.show()\n",
    "        \n",
    "if train_loss_history is not None and val_loss_history is not None:\n",
    "    plot_loss_history(train_loss_history, val_loss_history)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Example №1\n",
      "Generated sentence: Very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very highly detailed epic photo of face with venetian mask, intricate\n",
      "\n",
      "Example №2\n",
      "Generated sentence: portrait of a beautiful woman wearing red venetian mask, intricate, elegant, highly detailed, digital painting, artstation, concept art, by artgerm and greg rutkowski and alphonse mucha\n",
      "\n",
      "Example №3\n",
      "Generated sentence: a beautiful digital illustration painting of a detailed gothic fantasy castle and roots, dark mushroom, flowers by benoit b. mandelbrot, steven belledin, martin johnson heade, lee madgwick, caspar david friedrich, and david rios ferreira. 8 k resolution trending on artstation\n",
      "\n",
      "Example №4\n",
      "Generated sentence: a beautiful digital illustration painting of a detailed gothic fantasy castle and roots, dark mushroom, flowers by benoit b. mandelbrot, steven belledin, martin johnson heade, lee madgwick, caspar david friedrich, and david rios ferreira. 8 k resolution trending on artstation\n",
      "\n",
      "Example №5\n",
      "Generated sentence: portrait of cute little gothic girl, warhammer 40000, cyberpunk, intricate, elegant, highly detailed, digital painting, artstation, concept art, smooth, sharp focus, illustration, art by Krenz Cushart and Artem Demura and alphonse mucha\n",
      "\n",
      "Example №6\n",
      "Generated sentence: a highly detailed epic cinematic concept art CG render digital painting artwork: sophisticated pane ( from a man. By Greg Rutkowski, Ilya Kuvshinov, WLOP, Stanley Artgerm Lau, Ruan Jia and Fenghua Zhong, trending on ArtStation, made in Maya, Blender\n",
      "\n",
      "Example №7\n",
      "Generated sentence: an anthropomorphic fox fursona!!! by simon bisley and greg rutkowski, trending on artstation, unreal engine, 8 k\n",
      "\n",
      "Example №8\n",
      "Generated sentence: a highly detailed epic cinematic concept art CG render digital painting artwork: wide shot of sci - firealistic Portrait of Soviet dieselpunk star wars alex grey volumetric lighting. By Greg Rutkowski, in the style of Francis Bacon and Syd Mead and Norman Rockwell and Beksinski\n",
      "\n",
      "Example №9\n",
      "Generated sentence: an anthropomorphic fox man wearing a long black clothes, holding a scimer, holding a glowing La movie poster art by michael whelan, rossetti bouguereau, artgerm, retro, nostalgic, old fashioned\n",
      "\n",
      "Example №10\n",
      "Generated sentence: an anthropomorphic fox fursona wearing headphones, character design, trending on artstation, by neon colours, art by walking\n"
     ]
    }
   ],
   "source": [
    "sentences_num = 10\n",
    "\n",
    "# [Generate sentences]\n",
    "for i in range(sentences_num):\n",
    "    print(f\"\\nExample №{i + 1}\")\n",
    "    print(f\"Generated sentence: {predict()[0]}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}