Contextual word embeddings extraction.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "CNN-FakeNewsDetection.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "TkOVdKbFSzdV"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "## **Initial configuration**"
      ],
      "metadata": {
        "id": "vwnE79LIStJu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import tqdm\n",
        "import time\n",
        "\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "import pandas as pd\n",
        "from tensorflow import keras\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "from tensorflow.keras import layers\n",
        "from keras import backend as K\n",
        "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n",
        "from sklearn.metrics import f1_score, ConfusionMatrixDisplay\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "!pip install -q tf-models-official\n",
        "!pip install -q -U tensorflow-text \n",
        "!pip install -U spacy\n",
        "!python -m spacy download en_core_web_md\n",
        "\n",
        "import tensorflow_hub as hub\n",
        "import tensorflow_text as text\n",
        "\n",
        "import en_core_web_md\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        " \n",
        "!cp /content/drive/MyDrive/HLT/dataset/* ."
      ],
      "metadata": {
        "id": "B7SVtexZvMXR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "760f6df1-d413-494b-e663-c57b79dc811f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n",
            "cp: -r not specified; omitting directory '/content/drive/MyDrive/HLT/dataset/base_model'\n",
            "cp: -r not specified; omitting directory '/content/drive/MyDrive/HLT/dataset/dataset_split'\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "NUM_CLASSES=4\n",
        "\n",
        "MODEL_DIR=\"/content/drive/MyDrive/HLT/\"\n",
        "DATA_DIR=MODEL_DIR+\"dataset/dataset_split/\"\n",
        "\n",
        "BERT_MODEL=\"https://tfhub.dev/tensorflow/albert_en_base/3\"\n",
        "BERT_PREPROCESS_MODEL=\"https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3\"\n",
        "\n",
        "BATCH_SIZE=32\n",
        "LABELS=['agree', 'disagree', 'discuss', 'unrelated']\n",
        "LABELS_RELATED=['unrelated','related']\n",
        "RELATED=LABELS[0:3]\n",
        "TARGET_COLUMN=\"Stance\"\n",
        "\n",
        "MAX_WORDS=20000     # max number of words in the dictionary\n",
        "SEQ_LENGTH=128      # max sequence length of text \n",
        "EMBEDDING_DIM=300   # size of spacy embeddings"
      ],
      "metadata": {
        "id": "qa-GACwd8PSg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# utility functions\n",
        "def get_predictions(head, body, model_used):\n",
        "  \n",
        "  prediction = model_used.predict([head, body])\n",
        "\n",
        "  pred_labels = prediction.argmax(axis=1)\n",
        "  pred_labels = np.array([LABELS[i] for i in pred_labels])\n",
        "\n",
        "  return pred_labels\n",
        "\n",
        "\n",
        "def score_metrics(gold_labels, pred_labels):\n",
        "  score = 0.0\n",
        "  for i, (g, t) in enumerate(zip(gold_labels, pred_labels)):\n",
        "        g_stance, t_stance = g, t\n",
        "        if g_stance == t_stance:\n",
        "            score += 0.25\n",
        "            if g_stance != 'unrelated':\n",
        "                score += 0.50\n",
        "        if g_stance in RELATED and t_stance in RELATED:\n",
        "            score += 0.25\n",
        "\n",
        "  return score\n",
        "\n",
        "\n",
        "def get_vectorizer(seq_length):\n",
        "  headline, articleBody, labels = get_dataset(\"train\")\n",
        "  concat = np.concatenate((headline, articleBody))\n",
        "\n",
        "  vectorizer = layers.TextVectorization(max_tokens=MAX_WORDS, \n",
        "                                output_mode = 'int', \n",
        "                                standardize='lower_and_strip_punctuation',\n",
        "                                split=\"whitespace\",\n",
        "                                ngrams=None, \n",
        "                                output_sequence_length=seq_length,\n",
        "                                pad_to_max_tokens=True,)\n",
        "  \n",
        "  vectorizer.adapt(concat)\n",
        "\n",
        "  return vectorizer\n",
        "\n",
        "\n",
        "def play_with_vectorizer(vectorizer):\n",
        "  voc = vectorizer.get_vocabulary()\n",
        "  num_tokens = len(voc)\n",
        "  print(f'Vocabulary size: {num_tokens}')\n",
        "  print(f'Vocabulary beginning: {voc[:5]}')\n",
        "\n",
        "  example = \"the pen is on the table\"\n",
        "  vectorized_example = vectorizer([[example]])\n",
        "\n",
        "  print(f'Example to give as input to the vectorizer: \"{example}\"')\n",
        "  print(f'Size of vectorized example: {vectorized_example.numpy().shape}')\n",
        "  print(f'Vector of the example: {vectorized_example.numpy()[0]}')\n",
        "\n",
        "\n",
        "def get_embedding_matrix(vectorizer, embedding_dim):\n",
        "  voc = vectorizer.get_vocabulary()\n",
        "  num_tokens = len(voc)\n",
        "  word_index = dict(zip(voc, range(num_tokens)))   # buiilding a dictionary with a (key, value)=(word, position in voc) for each word in the vocabulary\n",
        "\n",
        "  nlp = en_core_web_md.load()   # loading pretrained embedding with spacy\n",
        "           \n",
        "  hits = 0\n",
        "  misses = 0\n",
        "\n",
        "  # Prepare embedding matrix\n",
        "  embedding_matrix = np.zeros((num_tokens, embedding_dim))  #  embedding_dim is the spacy dimensionality of embeddings\n",
        "\n",
        "  for word, i in tqdm.tqdm(word_index.items()):\n",
        "      embedding_vector = nlp(str(word)).vector\n",
        "      if np.linalg.norm(embedding_vector)!=0:     # Words not found in embedding index will be all-zeros, so their norm would be zero. \n",
        "          embedding_matrix[i] = embedding_vector  # This includes the representation for \"padding\" and \"OOV\" i.e. Out Of Vocabulary\n",
        "          hits += 1                               \n",
        "      else:\n",
        "          misses += 1\n",
        "  print()\n",
        "  print(\"Converted %d words (%d misses)\" % (hits, misses))\n",
        "\n",
        "  return embedding_matrix\n",
        "\n",
        "\n",
        "def get_dataset(dataset):\n",
        "  data_bodies = pd.read_csv(dataset + \"_bodies.csv\")\n",
        "  data_stances = pd.read_csv(dataset + \"_stances.csv\")\n",
        "\n",
        "  join_data =  pd.merge(data_stances, data_bodies, how=\"inner\", on=[\"Body ID\"])\n",
        "  join_data = join_data[['Body ID', 'Headline', 'articleBody', 'Stance']]\n",
        "\n",
        "  join_data[TARGET_COLUMN] = join_data[TARGET_COLUMN].astype('category')\n",
        "\n",
        "  headline = join_data.Headline.map(str).to_numpy()  # headline -> str\n",
        "  articleBody = join_data.articleBody.map(str).to_numpy()  # articleBody -> str\n",
        "\n",
        "\n",
        "  labels = join_data[TARGET_COLUMN].cat.codes.to_numpy()  # integer labels of categories\n",
        "\n",
        "  return headline, articleBody, labels\n",
        "\n",
        "\n",
        "def save_dataset_splitted(sample_size=-1):\n",
        "  data_bodies = pd.read_csv(\"train_bodies.csv\")\n",
        "  data_stances = pd.read_csv(\"train_stances.csv\")\n",
        "\n",
        "  join_data =  pd.merge(data_stances, data_bodies, how=\"inner\", on=[\"Body ID\"])\n",
        "  join_data = join_data[['Body ID', 'Headline', 'articleBody', 'Stance']]\n",
        "\n",
        "  join_data[TARGET_COLUMN] = join_data[TARGET_COLUMN].astype('category')\n",
        "\n",
        "  if sample_size != -1:\n",
        "    join_data = join_data[:sample_size]\n",
        "\n",
        "  train_annot, val_annot = train_test_split(join_data, test_size=0.2, random_state=42, shuffle=True, stratify=join_data[TARGET_COLUMN])\n",
        "\n",
        "  train_annot.to_csv(DATA_DIR + 'train_annot.csv')\n",
        "  val_annot.to_csv(DATA_DIR + 'val_annot.csv')\n",
        "  \n",
        "\n",
        "def get_dataset_splitted():\n",
        "  train_annot = pd.read_csv(DATA_DIR + \"train_annot.csv\")\n",
        "  val_annot = pd.read_csv(DATA_DIR + \"val_annot.csv\")\n",
        "\n",
        "  headline_train = train_annot.Headline.map(str).to_numpy()  # headline -> str\n",
        "  headline_val = val_annot.Headline.map(str).to_numpy()  \n",
        "  \n",
        "  train_annot[TARGET_COLUMN] = train_annot[TARGET_COLUMN].astype('category')\n",
        "  val_annot[TARGET_COLUMN] = val_annot[TARGET_COLUMN].astype('category')\n",
        "  articleBody_train = train_annot.articleBody.map(str).to_numpy()  # articleBody -> str\n",
        "  articleBody_val = val_annot.articleBody.map(str).to_numpy()  \n",
        "\n",
        "\n",
        "  labels_train = train_annot[TARGET_COLUMN].cat.codes.to_numpy()  # integer labels of categories\n",
        "  labels_val = val_annot[TARGET_COLUMN].cat.codes.to_numpy()\n",
        "\n",
        "  return headline_train, articleBody_train, labels_train, headline_val, articleBody_val, labels_val\n",
        "\n",
        "\n",
        "def get_confusion_matrix(gold_labels, pred_labels):  \n",
        "    confusion_matrix = [[0, 0, 0, 0],   #this is the confusion_matrix\n",
        "                        [0, 0, 0, 0],\n",
        "                        [0, 0, 0, 0],\n",
        "                        [0, 0, 0, 0]]\n",
        "\n",
        "    for i, (g, t) in enumerate(zip(gold_labels, pred_labels)):\n",
        "        confusion_matrix[LABELS.index(g)][LABELS.index(t)] += 1\n",
        "\n",
        "    return confusion_matrix\n",
        "\n",
        "\n",
        "# monitoring metrics\n",
        "def f1_training(y_true, y_pred):     \n",
        "    \"\"\" Calculate the micro f1, so globally by counting the total true positives, false negatives and false positives.\"\"\" \n",
        "    precision = precision_training(y_true, y_pred)\n",
        "    recall = recall_training(y_true, y_pred)\n",
        "    return 2*((precision*recall)/(precision+recall+K.epsilon()))\n",
        "\n",
        "\n",
        "def recall_training(y_true, y_pred):\n",
        "    \"\"\" Calculate the recall, so globally by counting the total true positives, false negatives and false positives.\"\"\" \n",
        "    y_true = K.ones_like(y_true) \n",
        "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
        "    all_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n",
        "    \n",
        "    recall = true_positives / (all_positives + K.epsilon())\n",
        "    return recall\n",
        "\n",
        "\n",
        "def precision_training(y_true, y_pred):\n",
        "    y_true = K.ones_like(y_true) \n",
        "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
        "    \n",
        "    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n",
        "    precision = true_positives / (predicted_positives + K.epsilon())\n",
        "    return precision"
      ],
      "metadata": {
        "id": "BbWkHwS1JDR7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## **CNN base**"
      ],
      "metadata": {
        "id": "TkOVdKbFSzdV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "vectorizer = get_vectorizer(SEQ_LENGTH)\n",
        "play_with_vectorizer(vectorizer)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5NEOCvqN7g8-",
        "outputId": "680eb845-2ce4-41d3-cad8-9e5bb35e5b38"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Vocabulary size: 20000\n",
            "Vocabulary beginning: ['', '[UNK]', 'the', 'to', 'a']\n",
            "Example to give as input to the vectorizer: \"the pen is on the table\"\n",
            "Size of vectorized example: (1, 128)\n",
            "Vector of the example: [   2    1    9   11    2 3038    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "embedding_matrix = get_embedding_matrix(vectorizer, EMBEDDING_DIM)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zExeeZOV7i_G",
        "outputId": "bafbce94-53a1-4620-d85d-d976bb5bd212"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 20000/20000 [01:44<00:00, 190.89it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Converted 17713 words (2287 misses)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#manipulate the dataset in input to the model\n",
        "headline_train, body_train, labels_train, headline_val, body_val, labels_val = get_dataset_splitted()\n",
        "\n",
        "vect_head_train = vectorizer(np.array([[s] for s in headline_train])).numpy()    # First create a numpy array of shape Nx1 with N = #texts, \n",
        "vect_head_val = vectorizer(np.array([[s] for s in headline_val])).numpy()        # then get a Tensor of shape NxSEQ_LENGTH that is the result of the vectorization\n",
        "                                                                                 # of all text, truncated at SEQ_LENGTH and finally convert it into a numpy array\n",
        "\n",
        "vect_body_train = vectorizer(np.array([[s] for s in body_train])).numpy()\n",
        "vect_body_val = vectorizer(np.array([[s] for s in body_val])).numpy()"
      ],
      "metadata": {
        "id": "RiyWwmha6AsN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Build the model with the Functional API\n",
        "\n",
        "# initialize an embedding layer that is just a matrix of size V x E\n",
        "# where is done a lookup operation to take the word embedding of a word\n",
        "embedding_layer = layers.Embedding(\n",
        "    len(vectorizer.get_vocabulary()),\n",
        "    EMBEDDING_DIM,\n",
        "    embeddings_initializer=keras.initializers.Constant(embedding_matrix),                                                                                  \n",
        "    trainable=False,                                                         \n",
        ")\n",
        "\n",
        "head_input = keras.Input(shape=(None,), dtype=\"int64\", name=\"title\")\n",
        "body_input = keras.Input(shape=(None,), dtype=\"int64\", name=\"body\")\n",
        "\n",
        "embedded_head = embedding_layer(head_input)\n",
        "embedded_body = embedding_layer(body_input)\n",
        "\n",
        "head = layers.Conv1D(256, 5, activation=\"relu\", padding=\"same\")(embedded_head)\n",
        "head = layers.Dropout(0.5)(head)\n",
        "head = layers.MaxPooling1D()(head)  # padding default to valid\n",
        "head = layers.Conv1D(256, 5, activation=\"relu\", padding=\"same\")(head)\n",
        "head = layers.Dropout(0.5)(head)\n",
        "head = layers.MaxPooling1D()(head)\n",
        "head = layers.Conv1D(512, 5, activation=\"relu\", padding=\"same\")(head)\n",
        "head = layers.Dropout(0.5)(head)\n",
        "head = layers.MaxPooling1D()(head)\n",
        "head = layers.Conv1D(512, 5, activation=\"relu\", padding=\"same\")(head)\n",
        "head = layers.Dropout(0.5)(head)\n",
        "head = layers.Conv1D(768, 5, activation=\"relu\", padding=\"same\")(head)\n",
        "head = layers.Dropout(0.5)(head)\n",
        "head = layers.GlobalMaxPooling1D()(head)\n",
        "\n",
        "body = layers.Conv1D(256, 5, activation=\"relu\", padding=\"same\")(embedded_body)\n",
        "body = layers.Dropout(0.5)(body)\n",
        "body = layers.MaxPooling1D()(body)\n",
        "body = layers.Conv1D(256, 5, activation=\"relu\", padding=\"same\")(body)\n",
        "body = layers.Dropout(0.5)(body)\n",
        "body = layers.MaxPooling1D()(body)\n",
        "body = layers.Conv1D(512, 5, activation=\"relu\", padding=\"same\")(body)\n",
        "body = layers.Dropout(0.5)(body)\n",
        "body = layers.MaxPooling1D()(body)\n",
        "body = layers.Conv1D(512, 5, activation=\"relu\", padding=\"same\")(body)\n",
        "body = layers.Dropout(0.5)(body)\n",
        "body = layers.Conv1D(768, 5, activation=\"relu\", padding=\"same\")(body)\n",
        "body = layers.Dropout(0.5)(body)\n",
        "body = layers.GlobalMaxPooling1D()(body) \n",
        "\n",
        "concatted = tf.keras.layers.Concatenate()([head, body])\n",
        "\n",
        "x = layers.Dense(1024, activation=\"relu\")(concatted)\n",
        "x = layers.Dense(1024, activation=\"relu\")(x)\n",
        "x = layers.Dense(1024, activation=\"relu\")(x)\n",
        "preds = layers.Dense(NUM_CLASSES, activation=\"softmax\", name='classifier')(x)\n",
        "\n",
        "model = keras.Model([head_input, body_input], preds)\n",
        "model.summary()"
      ],
      "metadata": {
        "id": "V8cxDPSR2wtt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tf.keras.utils.plot_model(model)"
      ],
      "metadata": {
        "id": "XJSvB48ZRev1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=False) #in fact labels are not one-hot encoded\n",
        "\n",
        "earlystopping = EarlyStopping(monitor='val_loss', mode='min', patience=5, verbose=1)\n",
        "\n",
        "checkpoint = ModelCheckpoint(MODEL_DIR + \"/cnn_model/seq_length_5.h5\",   \n",
        "        monitor='val_loss', mode='min', verbose=1, \n",
        "        save_best_only=True, save_freq='epoch'\n",
        ")\n",
        "\n",
        "learning_rate = 2e-4\n",
        "optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate)\n",
        "\n",
        "metrics = ['accuracy', \n",
        "          f1_training,\n",
        "          recall_training,\n",
        "          precision_training]\n",
        "\n",
        "model.compile(\n",
        "    loss=loss, \n",
        "    optimizer=optimizer, \n",
        "    metrics=metrics\n",
        ")"
      ],
      "metadata": {
        "id": "NuhTpxwy25mv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "EPOCHS=20\n",
        "\n",
        "model.fit({\"title\": vect_head_train, \"body\": vect_body_train},\n",
        "          labels_train,  \n",
        "          batch_size=BATCH_SIZE,\n",
        "          epochs=EPOCHS, \n",
        "          validation_data=({\"title\": vect_head_val, \"body\": vect_body_val}, labels_val), \n",
        "          callbacks=[checkpoint, earlystopping],\n",
        "          verbose=1\n",
        ")"
      ],
      "metadata": {
        "id": "fHSWHHFPj8oK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## **CNN with BERT embeddings**"
      ],
      "metadata": {
        "id": "wTzlizKfQyfB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "head_input = keras.Input(shape=(), dtype=tf.string, name=\"title\")\n",
        "body_input = keras.Input(shape=(), dtype=tf.string, name=\"body\")\n",
        "\n",
        "preprocessing_layer = hub.KerasLayer(BERT_PREPROCESS_MODEL, name='preprocessing')\n",
        "\n",
        "encoder_head_inputs = preprocessing_layer(head_input)\n",
        "encoder_body_inputs = preprocessing_layer(body_input)\n",
        "\n",
        "encoder = hub.KerasLayer(BERT_MODEL, trainable=False, name='BERT_encoder')\n",
        "encoder.trainable = False\n",
        "\n",
        "embedded_head = encoder(encoder_head_inputs)\n",
        "embedded_head = embedded_head['encoder_outputs'][-4:]\n",
        "embedded_head = layers.Concatenate(axis=2)(embedded_head)\n",
        "\n",
        "embedded_body = encoder(encoder_body_inputs)\n",
        "embedded_body = embedded_body['encoder_outputs'][-4:]\n",
        "embedded_body = layers.Concatenate(axis=2)(embedded_body)\n",
        "\n",
        "head = layers.Conv1D(256, 5, activation=\"relu\", padding=\"same\")(embedded_head)\n",
        "head = layers.Dropout(0.5)(head)\n",
        "head = layers.MaxPooling1D()(head)  # padding default to valid\n",
        "head = layers.Conv1D(256, 5, activation=\"relu\", padding=\"same\")(head)\n",
        "head = layers.Dropout(0.5)(head)\n",
        "head = layers.MaxPooling1D()(head)\n",
        "head = layers.Conv1D(512, 5, activation=\"relu\", padding=\"same\")(head)\n",
        "head = layers.Dropout(0.5)(head)\n",
        "head = layers.MaxPooling1D()(head)\n",
        "head = layers.Conv1D(512, 5, activation=\"relu\", padding=\"same\")(head)\n",
        "head = layers.Dropout(0.5)(head)\n",
        "head = layers.Conv1D(768, 5, activation=\"relu\", padding=\"same\")(head)\n",
        "head = layers.Dropout(0.5)(head)\n",
        "head = layers.GlobalMaxPooling1D()(head)\n",
        "\n",
        "body = layers.Conv1D(256, 5, activation=\"relu\", padding=\"same\")(embedded_body)\n",
        "body = layers.Dropout(0.5)(body)\n",
        "body = layers.MaxPooling1D()(body)\n",
        "body = layers.Conv1D(256, 5, activation=\"relu\", padding=\"same\")(body)\n",
        "body = layers.Dropout(0.5)(body)\n",
        "body = layers.MaxPooling1D()(body)\n",
        "body = layers.Conv1D(512, 5, activation=\"relu\", padding=\"same\")(body)\n",
        "body = layers.Dropout(0.5)(body)\n",
        "body = layers.MaxPooling1D()(body)\n",
        "body = layers.Conv1D(512, 5, activation=\"relu\", padding=\"same\")(body)\n",
        "body = layers.Dropout(0.5)(body)\n",
        "body = layers.Conv1D(768, 5, activation=\"relu\", padding=\"same\")(body)\n",
        "body = layers.Dropout(0.5)(body)\n",
        "body = layers.GlobalMaxPooling1D()(body) \n",
        "\n",
        "concatted = layers.Concatenate()([head, body])\n",
        "\n",
        "x = layers.Dense(1024, activation=\"relu\")(concatted) \n",
        "x = layers.Dense(1024, activation=\"relu\")(x)\n",
        "x = layers.Dense(1024, activation=\"relu\")(x) \n",
        "preds = layers.Dense(NUM_CLASSES, activation=\"softmax\", name='classifier')(x)\n",
        "\n",
        "model_bert = keras.Model([head_input, body_input], preds)\n",
        "model_bert.summary()"
      ],
      "metadata": {
        "id": "OU1RUtKdQ3no",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bd0e3ccb-72e4-4854-d993-09eeb62480f0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_1\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " title (InputLayer)             [(None,)]            0           []                               \n",
            "                                                                                                  \n",
            " body (InputLayer)              [(None,)]            0           []                               \n",
            "                                                                                                  \n",
            " preprocessing (KerasLayer)     {'input_mask': (Non  0           ['title[0][0]',                  \n",
            "                                e, 128),                          'body[0][0]']                   \n",
            "                                 'input_word_ids':                                                \n",
            "                                (None, 128),                                                      \n",
            "                                 'input_type_ids':                                                \n",
            "                                (None, 128)}                                                      \n",
            "                                                                                                  \n",
            " BERT_encoder (KerasLayer)      {'default': (None,   11683584    ['preprocessing[0][0]',          \n",
            "                                768),                             'preprocessing[0][1]',          \n",
            "                                 'sequence_output':               'preprocessing[0][2]',          \n",
            "                                 (None, 128, 768),                'preprocessing[1][0]',          \n",
            "                                 'encoder_outputs':               'preprocessing[1][1]',          \n",
            "                                 [(None, 128, 768),               'preprocessing[1][2]']          \n",
            "                                 (None, 128, 768),                                                \n",
            "                                 (None, 128, 768),                                                \n",
            "                                 (None, 128, 768),                                                \n",
            "                                 (None, 128, 768),                                                \n",
            "                                 (None, 128, 768),                                                \n",
            "                                 (None, 128, 768),                                                \n",
            "                                 (None, 128, 768),                                                \n",
            "                                 (None, 128, 768),                                                \n",
            "                                 (None, 128, 768),                                                \n",
            "                                 (None, 128, 768),                                                \n",
            "                                 (None, 128, 768)],                                               \n",
            "                                 'pooled_output': (                                               \n",
            "                                None, 768)}                                                       \n",
            "                                                                                                  \n",
            " concatenate_3 (Concatenate)    (None, 128, 3072)    0           ['BERT_encoder[0][9]',           \n",
            "                                                                  'BERT_encoder[0][10]',          \n",
            "                                                                  'BERT_encoder[0][11]',          \n",
            "                                                                  'BERT_encoder[0][12]']          \n",
            "                                                                                                  \n",
            " concatenate_4 (Concatenate)    (None, 128, 3072)    0           ['BERT_encoder[1][9]',           \n",
            "                                                                  'BERT_encoder[1][10]',          \n",
            "                                                                  'BERT_encoder[1][11]',          \n",
            "                                                                  'BERT_encoder[1][12]']          \n",
            "                                                                                                  \n",
            " conv1d_10 (Conv1D)             (None, 128, 256)     3932416     ['concatenate_3[0][0]']          \n",
            "                                                                                                  \n",
            " conv1d_15 (Conv1D)             (None, 128, 256)     3932416     ['concatenate_4[0][0]']          \n",
            "                                                                                                  \n",
            " dropout_10 (Dropout)           (None, 128, 256)     0           ['conv1d_10[0][0]']              \n",
            "                                                                                                  \n",
            " dropout_15 (Dropout)           (None, 128, 256)     0           ['conv1d_15[0][0]']              \n",
            "                                                                                                  \n",
            " max_pooling1d_6 (MaxPooling1D)  (None, 64, 256)     0           ['dropout_10[0][0]']             \n",
            "                                                                                                  \n",
            " max_pooling1d_9 (MaxPooling1D)  (None, 64, 256)     0           ['dropout_15[0][0]']             \n",
            "                                                                                                  \n",
            " conv1d_11 (Conv1D)             (None, 64, 256)      327936      ['max_pooling1d_6[0][0]']        \n",
            "                                                                                                  \n",
            " conv1d_16 (Conv1D)             (None, 64, 256)      327936      ['max_pooling1d_9[0][0]']        \n",
            "                                                                                                  \n",
            " dropout_11 (Dropout)           (None, 64, 256)      0           ['conv1d_11[0][0]']              \n",
            "                                                                                                  \n",
            " dropout_16 (Dropout)           (None, 64, 256)      0           ['conv1d_16[0][0]']              \n",
            "                                                                                                  \n",
            " max_pooling1d_7 (MaxPooling1D)  (None, 32, 256)     0           ['dropout_11[0][0]']             \n",
            "                                                                                                  \n",
            " max_pooling1d_10 (MaxPooling1D  (None, 32, 256)     0           ['dropout_16[0][0]']             \n",
            " )                                                                                                \n",
            "                                                                                                  \n",
            " conv1d_12 (Conv1D)             (None, 32, 512)      655872      ['max_pooling1d_7[0][0]']        \n",
            "                                                                                                  \n",
            " conv1d_17 (Conv1D)             (None, 32, 512)      655872      ['max_pooling1d_10[0][0]']       \n",
            "                                                                                                  \n",
            " dropout_12 (Dropout)           (None, 32, 512)      0           ['conv1d_12[0][0]']              \n",
            "                                                                                                  \n",
            " dropout_17 (Dropout)           (None, 32, 512)      0           ['conv1d_17[0][0]']              \n",
            "                                                                                                  \n",
            " max_pooling1d_8 (MaxPooling1D)  (None, 16, 512)     0           ['dropout_12[0][0]']             \n",
            "                                                                                                  \n",
            " max_pooling1d_11 (MaxPooling1D  (None, 16, 512)     0           ['dropout_17[0][0]']             \n",
            " )                                                                                                \n",
            "                                                                                                  \n",
            " conv1d_13 (Conv1D)             (None, 16, 512)      1311232     ['max_pooling1d_8[0][0]']        \n",
            "                                                                                                  \n",
            " conv1d_18 (Conv1D)             (None, 16, 512)      1311232     ['max_pooling1d_11[0][0]']       \n",
            "                                                                                                  \n",
            " dropout_13 (Dropout)           (None, 16, 512)      0           ['conv1d_13[0][0]']              \n",
            "                                                                                                  \n",
            " dropout_18 (Dropout)           (None, 16, 512)      0           ['conv1d_18[0][0]']              \n",
            "                                                                                                  \n",
            " conv1d_14 (Conv1D)             (None, 16, 768)      1966848     ['dropout_13[0][0]']             \n",
            "                                                                                                  \n",
            " conv1d_19 (Conv1D)             (None, 16, 768)      1966848     ['dropout_18[0][0]']             \n",
            "                                                                                                  \n",
            " dropout_14 (Dropout)           (None, 16, 768)      0           ['conv1d_14[0][0]']              \n",
            "                                                                                                  \n",
            " dropout_19 (Dropout)           (None, 16, 768)      0           ['conv1d_19[0][0]']              \n",
            "                                                                                                  \n",
            " global_max_pooling1d_2 (Global  (None, 768)         0           ['dropout_14[0][0]']             \n",
            " MaxPooling1D)                                                                                    \n",
            "                                                                                                  \n",
            " global_max_pooling1d_3 (Global  (None, 768)         0           ['dropout_19[0][0]']             \n",
            " MaxPooling1D)                                                                                    \n",
            "                                                                                                  \n",
            " concatenate_5 (Concatenate)    (None, 1536)         0           ['global_max_pooling1d_2[0][0]', \n",
            "                                                                  'global_max_pooling1d_3[0][0]'] \n",
            "                                                                                                  \n",
            " dense_3 (Dense)                (None, 1024)         1573888     ['concatenate_5[0][0]']          \n",
            "                                                                                                  \n",
            " dense_4 (Dense)                (None, 1024)         1049600     ['dense_3[0][0]']                \n",
            "                                                                                                  \n",
            " dense_5 (Dense)                (None, 1024)         1049600     ['dense_4[0][0]']                \n",
            "                                                                                                  \n",
            " classifier (Dense)             (None, 4)            4100        ['dense_5[0][0]']                \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 31,749,380\n",
            "Trainable params: 20,065,796\n",
            "Non-trainable params: 11,683,584\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#save_dataset_splitted()\n",
        "headline_train, body_train, labels_train, headline_val, body_val, labels_val = get_dataset_splitted()"
      ],
      "metadata": {
        "id": "_Zvsn7JVSg08"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=False) #in fact labels are not one-hot encoded\n",
        "\n",
        "earlystopping = EarlyStopping(monitor='val_loss', mode='min', patience=3, verbose=1)\n",
        "\n",
        "checkpoint = ModelCheckpoint(MODEL_DIR + \"/cnn_model/bert_large_test_drive.h5\",   \n",
        "        monitor='val_loss', mode='min', verbose=1, \n",
        "        save_best_only=True, save_freq='epoch'\n",
        ")\n",
        "\n",
        "learning_rate = 2e-4\n",
        "optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate)\n",
        "\n",
        "metrics = ['accuracy', \n",
        "          f1_training,\n",
        "          recall_training,\n",
        "          precision_training]\n",
        "\n",
        "model_bert.compile(\n",
        "    loss=loss, \n",
        "    optimizer=optimizer, \n",
        "    metrics=metrics\n",
        ")"
      ],
      "metadata": {
        "id": "RPL7jb0kSlGR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "EPOCHS=8\n",
        "\n",
        "model_bert.fit({\"title\": headline_train, \"body\": body_train},\n",
        "          labels_train,  \n",
        "          batch_size=BATCH_SIZE,\n",
        "          epochs=EPOCHS, \n",
        "          validation_data=({\"title\": headline_val, \"body\": body_val}, labels_val), \n",
        "          callbacks=[checkpoint, earlystopping],\n",
        "          verbose=1\n",
        ")   "
      ],
      "metadata": {
        "id": "5rHG7-VKSq-t"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model_bert.save(MODEL_DIR + \"/cnn_model/bert_large_4_epochs.h5\")"
      ],
      "metadata": {
        "id": "E25TF1HV2c8a"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## **Load and Test**"
      ],
      "metadata": {
        "id": "YZEu0aF6C8_z"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "MODEL_TO_LOAD = MODEL_DIR + \"cnn_model/seq_length_5.h5\"# CNN experiments/cnn_net_128_5_2048.h5\" #cnn_net_128_3_stratified  cnn_net_3_layers_2048 cnn_net_window_5\n",
        "model_loaded = tf.keras.models.load_model((MODEL_TO_LOAD), custom_objects={'KerasLayer':hub.KerasLayer, 'f1_training':f1_training, 'recall_training':recall_training, 'precision_training':precision_training,})"
      ],
      "metadata": {
        "id": "ZIznlB-pC4rA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "###########   TEST\n",
        "# compute score on test\n",
        "headline_test, articleBody_test, gold_labels = get_dataset(\"competition_test\")\n",
        "\n",
        "BERT = False\n",
        "\n",
        "if BERT:\n",
        "  start_time = time.time()\n",
        "  pred_labels = get_predictions(headline_test, articleBody_test, model_loaded)\n",
        "  inference_time = time.time() - start_time\n",
        "else:\n",
        "  vectorizer = get_vectorizer(SEQ_LENGTH)\n",
        "  vect_head_test = vectorizer(np.array([[s] for s in headline_test])).numpy()\n",
        "  vect_body_test = vectorizer(np.array([[s] for s in articleBody_test])).numpy()\n",
        "\n",
        "  start_time = time.time()\n",
        "  pred_labels = get_predictions(vect_head_test, vect_body_test, model_loaded)\n",
        "  inference_time = time.time()\n",
        "\n",
        "gold_labels = np.array([LABELS[i] for i in gold_labels])\n",
        "print(f'Inference time: {inference_time}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QeLVSpBBYelY",
        "outputId": "31df1fa4-936d-4b6b-de1d-e47f6f0eb351"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Inference time: 1643631194.4119856\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "fig, ax = plt.subplots(figsize=(6, 6))\n",
        "ConfusionMatrixDisplay.from_predictions(gold_labels, pred_labels, xticks_rotation='vertical', ax=ax)\n",
        "score_test = score_metrics(gold_labels, pred_labels)\n",
        "print(get_confusion_matrix(gold_labels, pred_labels))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 415
        },
        "id": "WfbdtyrS-g5Z",
        "outputId": "841152a4-5e32-4798-b1db-ce230e591c95"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[1335, 0, 469, 99], [490, 0, 141, 66], [1477, 0, 2606, 381], [4721, 39, 3945, 9644]]\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAF8CAYAAACJym8NAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd5wU9f3H8dfnKhzt6L2LKBoUpdqCYu9do0aNBXsnUaOJxqjRX+wtiVFjj91obKAoEhFRUEBpgvTO0fuV/fz+mAGOdpzH3c7s3vv5eOzDnZmd2c+sx372+/1+5jvm7oiIiCRbRtQBiIhI9aQEJCIikVACEhGRSCgBiYhIJJSAREQkEkpAIiISiayoAxARke074uBavnhJSYX3HzV2/UB3P7ISQ6o0SkAiIjFWsKSEEQNbVXj/7OY/NarEcCqVuuBERCQSagGJiMSaU+KJqIOoEkpAIiIx5kCC9JwyTQlIRCTmEqRnC0hjQCIiEgm1gEREYsxxStL0rgVKQCIiMacxIBERSToHSpSAREQkCunaAlIRgoiIREItIBGRGHNQEYKIiEQjPa8CUgISEYk1x1WEICIiEXAoSc/8oyIEERGJhlpAIiIxFkxGmp6UgEREYs0owaIOokooAYmIxJgDCY0BiYiIVB61gEREYk5dcCIiknTBZKRKQCIiEoGEKwGJiEiSpXMLSEUIIiISCbWARERizDFK0rStoAQkIhJzGgMSEZGkS+cxICWgnyknK89r5uRHHUYs+Np1UYcQG5aZGXUIseElJVGHEBsrWVrg7o137ihGiasLToCaOfn07nxR1GHEQmLMhKhDiI3MevWjDiE2SpaviDqE2Pik5NUZUccQZ0pAIiIxFsyGrRaQiIhEQGNAIiKSdO7pOwaUnmclIiKxpxaQiEjMJdQFJyIiyRZcB5SenVVKQCIisZa+Y0BKQCIiMZbOZdjpeVYiIhJ7agGJiMRciSYjFRGRZNPtGEREJDIJFSGIiEiypXMZdnqelYiIxJ5aQCIiMeaYihBERCQa6XodkBKQiEiMuZO2MyGk51mJiEjsqQUkIhJrptmwRUQk+Zz07YJTAhIRiTldByQiIknnGAmv+KM8zOw6MxtnZj+Y2b/NrIaZtTezEWY2xcxeNbOc8LW54fKUcHu7Use5OVw/ycyO2NH7KgGJiFRjZtYSuBro7u57ApnAmcC9wIPuvguwFLgw3OVCYGm4/sHwdZhZl3C/PYAjgSfMLLOs91YCEhGJuRIyKvwopyygppllAXnAPOAQ4I1w+3PAieHzE8Jlwu39zMzC9a+4+3p3nwZMAXru6E1FRCSmnJ2ejLSRmY0stfykuz+58fjuc8zsPmAmsBYYBIwClrl7cfiy2UDL8HlLYFa4b7GZLQcahuu/KvU+pffZJiUgEZFYM0p2rgy7wN27b/foZvUJWi/tgWXA6wRdaFVOXXAiItXbocA0d1/k7kXAW8D+QH7YJQfQCpgTPp8DtAYIt9cDFpdev419tkktoJi47roR9Ow5l2XLanDZZUcB8Otfj6VPnzkkEsby5bncf39vliypSe/eszn33O9JJIySEuPJJ/dh3LjGALz33qtMn14PgEWL8vjTnw6K7JyqWve+K7j0z3PJzHA+/HcDXnusadQhJUVGhvPwa6NYvCCH26/oCjjnXj2NA49YREmJ8cGrLXj3pVbUrlvEtX+eRPPWaykszOChWzszY0rtqMOvEideuJCjflWAGXz4ciPefroJHXZfw1X3zKJmrRIWzMrh3qvas2ZVmWPisVQJXXA7MhPobWZ5BF1w/YCRwGfAqcArwHnAO+Hr3w2Xh4fbP3V3N7N3gZfN7AGgBdAJ+LqsN1YCiomPP27Pu+92YsCAERvXvfnm7rzwQlcAjj/+R8466wcee6wHo0c35auvWgJGu3bL+P3vh9G//zEAFBZmcuWVSWk9Ryojw7ni7jncfGYHCuZl8+gHk/lqYD1mTq4RdWhV7oRfz2bW1DzyagXd84edOJ/GzdbT/9ieuBv1GhQCcPrFM5k6sTZ3XrMnrdqv5vJbJ/P7C/eOMvQq0bbzWo76VQFXH7sbRUXG3S9OYcTgulz715n8886WfP9VHQ4/o4BTL13A8/e1iDrcCtnJLrgyufsIM3sD+BYoBr4DngTeB14xszvDdU+HuzwNvGBmU4AlBJVvuPs4M3sNGB8e5wp3LynrvdO+C65UEzLWfvihCStX5my2bs2a7I3Pa9QohvCPcN267I3Pa9QoxtN0qvaydO62hrnTc5g/M5fiogyGvJNPnyOWRx1WlWvYdB09DlrMwDebb1x39JlzefnvbTf+HSxfEvwdtem4mjEj8gGYPa0WTVusI79hYfKDrmJtdlnHxNG1WL8ug0SJMfar2ux/1DJadVjH918FLb7vhtblgKOXRRxpxbgbCc+o8KN87+G3uftu7r6nu/86rGSb6u493X0Xdz/N3deHr10XLu8Sbp9a6jh3uXtHd+/s7h/u6H1T4ssZwMz+Q9C/WAN42N2fNLMLgRsJBs7GAOvd/UozexZYB3QDhpnZ48DjQGNgDXCxu080s8bA34E24dtc6+7DknleO3LeeWPp128aq1fncNNNB29cv99+szn//DHk56/nj3/c1M2Wk1PCww8PJJHI4LXXdmf48FZRhF3lGjYrYtHcTQm7YF42u+2zJsKIkuOSm6bwzP0dqVmreOO65q3XctCRi9iv3yKWL83h73fvwtyZeUybVJv9Ditg3Lf57PqLFTRpsY5GTdezbHFOGe+QeqZPqsH5N86lTn4xhesy6HHICiaPzWPGjzXpc8Ryhg/M58Bjl9K4Reom33SdiieVzuoCd98X6A5cHV489QegN8GA2W5bvL4VsJ+7X0/QnLwq3H8A8ET4mocJLrTqAZwCPLWtNzaz/mY20sxGFhYn90vuuee6cu65J/DZZ2057rjJG9d/+WUr+vc/hjvuOIBzz/1+4/rzzjuOa645gnvv7cMll3xL8+YrkxqvVJ2evyxg2ZIcpoyvs9n67JwEheszuOaM7nz0RnOuvXMSAK891YbadYp59M1vOP6sOfw0sQ6JRBSRV61ZU2ry2hNN+cvLk7nrxSlMHVeTRAk8cENbjjt3EY99MIGatRMUF1W/noK4S5kWEEHSOSl83hr4NfC5uy8BMLPXgV1Lvf51dy8xs9rAfsDrwbVSAOSG/z0U6FJqfV0zq+3uq0q/cVgz/yRAvbwWXrmnVT6ffdaWO+4Yyosv/mKz9T/80IRmzUZQt+56VqzIZfHiPADmz6/N2LFN6NhxGfPm1dnWIVPa4vnZm/2ibdS8iIJ52WXskfq6dFtB774F9DhwMdm5CfJqlTDgnvEUzM/ly08aAfDlJ4247s6JAKxdncWDt274Xeb8a9BXzJtVM6Loq9bAVxox8JXgM/jNjXNYNC+HWT/V4PdndwKgZft19OqXml20Dmk7G3ZKtIDMrC9Bsujj7nsRDIhN3MFuq8P/ZhBcULV3qcfupbb1LrW+5ZbJJ0otWmxqvfTpM4fZs4NEErRqgjzYseMSsrMTrFiRQ+3ahWRnB2N+deuup0uXAmbOrJv0uJNh0ug8WrYvpGnr9WRlJ+h7wjK+GlQv6rCq1LMPdeDcfvvxm8P7cO+ALowdkc99N3Vh+KeN6NozGN/4RY9lzJkR/AipVaeIrOygyXPEqfP4YWQ+a1en0m/O8qvXsAiAxi0K2f+oZXz2n/ob15k5Z10zn/deaBRliDvBKPGMCj/iLFX+GusRzD20xsx2I+h2qwX8MryIaiVBF9r3W+7o7ivMbJqZnebur4dTRnR19zEEV/xeBfwVwMz2dvfRSTqnzdx445d07bqQunXX88IL7/DCC3vSo8c8WrVaiTssXFiLRx8NriU74IDZ9Os3jeLiDAoLM7nnnv0Ao3XrFVx11Te4G2bOa6/tzsyZ6fmlnCgxHr+lJXe/PJWMTBj0SgNm/Jj+FXDb8vpTbfjtvRM46dzZrF2TycN/7AxA6w5ruOHuibjDjCm1Nq5PR398cip16pdQUmw8dktrVq/I4sQLF3LceYsAGPZhPoNebRhxlBUTlGGnZwvI3CPpUfpZzCwX+A/QDpgE5AO3E3S5/ZagFHAiMNvdbwmLEN5z9zfC/dsDfwOaA9kE8xXdYWaNCIoTdidIxkPd/dKyYqmX18J7d76osk8xJSXGTIg6hNjIrF8/6hBio2T5iqhDiI1PSl4dVdYsBOXRfI/6ft6/+1V4/3v3enOnY6gqKdECCsv/jtpyvZmNDKvhsoC3CZIU7n7+FvtPYxtTS7h7AXBGVcQsIiJlS4kEVIbbzexQgtLsQYQJSEQkXWy4H1A6SukE5O4Doo5BRKSqJVKjXuxnS+kEJCKS7tyhJE1bQOmZVkVEJPbUAhIRiTmNAYmISNIFRQjp2VmlBCQiEnNVeTuGKCkBiYjEWDrPhJCe7ToREYk9tYBERGJNY0AiIhKRdL0dgxKQiEiMpfOFqEpAIiIxl65dcOl5ViIiEntqAYmIxJhmwxYRkcioCEFERJJOF6KKiIhUMrWARERiLl2r4JSARETizFWEICIiEXBUhCAiIhFJ1xZQenYsiohI7KkFJCISY+lchq0EJCISc0pAIiKSdJqKR0REIpOuVXAqQhARkUioBfRzFRVhsxdEHYXETGLlyqhDiA9PRB1BenGNAYmISARUBSciIpFJ1wSkMSAREYmEWkAiIjGmMmwREYmMKwGJiEgU0vU6ICUgEZEY8zQuw1YRgoiIREItIBGRmNMYkIiIREBVcCIiEhG1gEREJOnSeSoeFSGIiEgk1AISEYkzD0qx05ESkIhIzOlCVBERSTonfYsQNAYkIiKRUAtIRCTWdB2QiIhEREUIIiISiXQdA1ICEhGJMff0TUAqQhARkUgoAYmIxFzCrcKP8jCzfDN7w8wmmtkEM+tjZg3M7GMzmxz+t374WjOzR8xsipmNNbN9Sh3nvPD1k83svB29rxKQiEjMuVf8UU4PAx+5+27AXsAE4CZgsLt3AgaHywBHAZ3CR3/gbwBm1gC4DegF9ARu25C0tkcJSEQk5tytwo8dMbN6wEHA08F7eaG7LwNOAJ4LX/YccGL4/ATgeQ98BeSbWXPgCOBjd1/i7kuBj4Ejy3pvJSARkRhzKp58ylm80B5YBPzLzL4zs6fMrBbQ1N3nha+ZDzQNn7cEZpXaf3a4bnvrt0sJSEQkvTUys5GlHv232J4F7AP8zd27AavZ1N0GgLs7waxAlUpl2CIiMbeT3/wF7t69jO2zgdnuPiJcfoMgAS0ws+buPi/sYlsYbp8DtC61f6tw3Ryg7xbrh5QVmFpAMZaR4Tz62jfc/thYAPbquZRHXv2GJ976muvvnEBGZiJ8pXPJTT/y1Ptf8fibX9Nx95XRBZ1E3fuu4Kn/TeRfwyZw+pULog6nyl331+m88u0Y/v7xuK22nXzxAj6aOYq69YsBaNVxHQ++PZF3J3/LKf3nJzvUpKtVt5hbn5zGU59P4J9DJrD7vqsBOP43i3jq8wk8+elELrxlbsRRVpBX7RiQu88HZplZ53BVP2A88C6woZLtPOCd8Pm7wLlhNVxvYHnYVTcQONzM6ofFB4eH67Yr0haQmd0OrALqAkPd/ZMo44mbE86ZxaxpeeTVKsHMuf6uCfz+or2ZMyOPc66YyqHHz2fQ2y3ofuASWrZdy0XH9KJz1xVceeskrju7rB88qS8jw7ni7jncfGYHCuZl8+gHk/lqYD1mTq4RdWhV5uPXG/Lf55ow4MFpm61v1LyQfQ9awYLZORvXrVyWyd9ua02fI5YlO8xIXHbHHEZ+Vpc7+7cnKztBbs0Ee+23kv2OWM5lh3WmqDCDeg2Log6z4qp+Kp6rgJfMLAeYCvyGoIHympldCMwATg9f+wFwNDAFWBO+FndfYmZ/Br4JX3eHuy8p601j0QJy9z9WVfIJs3QszvPnaNh0HT0OXMzAN1sAUCe/iOIiY86MPAC+G96A/Q9bBEDvgwsY/G4zwJg0th616hRTv9H6qEJPis7d1jB3eg7zZ+ZSXJTBkHfy6XPE8qjDqlI/fF2Hlcsyt1p/yW2zeOrulpt9SS1fnM2PY2tRUpyeV9CXllenhF/0Ws1H/24AQHFRBqtXZHHsuYt59fGmFBUG//yXL86OMsxYc/fR7t7d3bu6+4nuvtTdF7t7P3fv5O6HbkgmYfXbFe7e0d1/4e4jSx3nGXffJXz8a0fvm/QvZjO7xcx+NLMvgM7humfN7NTw+T1mNj68wOm+cN1xZjYirND4xMyahusbhxdIjQsrN2aYWSMza2dmk8zseeAHoLWZ/dbMvgmP+6dS8ZxjZl+b2Wgz+4eZbf0vPAKX/G4Kzzy4C4mwl23F0mwyM51OXVYAcMBhi2jcLEgyjZqsZ9H83I37FizIpVGT9E5ADZsVsWjupl/8BfOyadQ8hX/hVlDvw5axeH4O0ybkRR1KZJq1Wc/yxVnc8OBMHh84iWv/OpPcmiW07LCOPXuu4uH//shf35jMrnutiTrUCqviKrjIJDUBmdm+wJnA3gRNuB5bbG8InATs4e5dgTvDTV8AvcMKjVeA34XrbwM+dfc9CAbO2pQ6XCfgiXBb53C5Z/je+5rZQWa2O3AGsL+77w2UAGdvI+7+GypIChPrdvZj2KGeBxWwbEk2U8bXKR0F9/xuDy7+3RQefHkka9dkUlIS7z8uqVq5NRKceeU8nr+/RdShRCozE3b5xRree74RVxzRmXVrMjjjyoVkZkKd/BKuOa4TT93Zglv+Pp1k9GVVhSRciBqJZI8BHQi87e5rAMzs3S22LwfWAU+b2XvAe+H6VsCrYSVGDrChE/wAgoSFu39kZktLHWtGeJEUBINhhwPfhcu1CRJSV2Bf4BszA6jJpkqPjdz9SeBJgHrZjav8f2mXbsvpffBiehw4nOzcBHm1ihnwl/Hcd3MXfnd+MOtFtz5LaNk2+EVXsDB3Y2sIoFHT9RQszN3msdPF4vnZNG5RuHG5UfMiCuZVry6W5m3X06x1IX/7aDwQjAU99sF4rjl+d5Yuqj6fRcG8bBbNy2bSd7UA+OL9fE6/ciEF87IZ9mE9wJg0uhaJBNRrUMLyJalV/JvOd0SN1f8Jdy82s54EVRinAlcChwCPAg+4+7tm1he4vRyHW13quQF/cfd/lH6BmV0FPOfuN1dC+JXm2Yc78uzDHQH4RfelnHL+LO67uQv1GhSyfEkOWdkJTrtgBq/+sx0AIz5ryHFnzeHzD5vQuesKVq/KYmlBeiegSaPzaNm+kKat17N4fjZ9T1jGPVe0jTqspJo+qSZn7rPXxuXnhn3PVcfuzoqlsfpnXeWWLsqmYG4OrTquY/ZPNdj7gJXM/DGXeTNy2Wu/VYz5sg4tO6wjO8dZviQWPew/jwNKQJViKPCsmf0lfO/jgI1JwcxqA3nu/oGZDSOoxgCoR1BjDpvKAgGGEVRm3GtmhwPbm3doIPBnM3vJ3VeZWUugiGB+o3fM7EF3XxjOZVTH3WdUytlWslPOn0nPXy4mw5z3X2vJmK+D0/3mfw3pcdASnv7gK9avy+TBW3eLONKqlygxHr+lJXe/PJWMTBj0SgNm/Ji+FXAANz06la59VlK3fjEvjBjLiw+0YOCrjbb52vqNi3jkvQnk1S7BE8aJFy7kkn57sGZVCn4Bl8Pjf2jJjY/OICvbmT8zh/uvb8O6NRlcf/8s/jF4IkVFxl+vbUPwW1TiwjzJnYRmdgtBElkIzAS+BfYk6G4bRlBrXoPgL+U+d3/OzE4AHgSWAp8CPdy9r5k1Af5NMEXEcOBYoB3QHHjP3fcs9b7XABeFi6uAc9z9JzM7A7iZYDysCLiiVNfdVuplN/Y++SdXxkeR8koWl1lhWa1YVvVqdZTFS0qiDiE2Pkm8PmoHF4HuUG6Hlt7yrisqvP+0s27Z6RiqStL/1bj7XcBdZbyk5zb2eYdNF0GVthw4Iuy660OQmNYD0wmSWuljPEww4+uWx34VeLXcJyAikmwxLyaoqFT/2daG4EKpDKAQuDjieEREKln8y6krKqUTkLtPBrpFHYeISJVK0xZQys0QICIi6SGlW0AiImnPdR2QiIhEJU274JSARERir5q1gMzsUcrIu+5+dZVEJCIi1UJZLaCRZWwTEZFkqW5dcO7+XOllM8vbMImoiIgkUZomoB2WYZtZHzMbD0wMl/cysyeqPDIREdk0GWlFHzFWnuuAHgKOABYDuPsY4KCqDEpERDZJ1/sBletCVHeftcUqzTYoIiI7pTxl2LPMbD/AzSwbuAaYULVhiYjIRjFvyVRUeRLQpQSzSLcE5hLcW6fic4OLiMjPE/OxnIraYQJy9wLg7CTEIiIi22Bp2gIqTxVcBzP7r5ktMrOFZvaOmXVIRnAiIpK+ylOE8DLwGsFdRlsArxPchVRERKqa7+QjxsqTgPLc/QV3Lw4fLxLcMltERKrcTlwDFPOxo7LmgmsQPv3QzG4CXiHIp2cAHyQhNhERgdi3ZCqqrCKEUQSnvSGFXlJqmwM3V1VQIiJSSnVLQO7ePpmBiIhI9VKu+wGZ2Z5AF0qN/bj781UVlIiIlFLdWkAbmNltQF+CBPQBcBTwBaAEJCJS1TZMRpqGylMFdyrQD5jv7r8B9gLqVWlUIiKykXnFH3FWni64te6eMLNiM6sLLARaV3FcIiKyQcwTSUWVJwGNNLN84J8ElXGrgOFVGpWIiKS98swFd3n49O9m9hFQ193HVm1YIiKS7sq6EHWfsra5+7dVE5KIiJQW97GciiqrBXR/GdscOKSSY0kJnpNNon2LqMOIh8VLoo4gNlae3D3qEGKj3kfjow4hPpZX0nHStAqurAtRD05mICIiUr2U60JUERGJSArMal1RSkAiInGnBCQiIlFI1yKE8twR1czsHDP7Y7jcxsx6Vn1oIiICVOsb0j0B9AF+FS6vBB6vsohERKRaKE8XXC9338fMvgNw96VmllPFcYmIyAYxb8lUVHkSUJGZZRJ+BGbWGEhUaVQiIgKkxqSiFVWeBPQI8DbQxMzuIpgd+9YqjUpERDapbheibuDuL5nZKIJbMhhwortPqPLIREQkUF1bQGbWBlgD/Lf0OnefWZWBiYhIeitPF9z7BPnXCG7J3R6YBOxRhXGJiEio2o4BufsvSi+Hs2Rfvp2Xi4hIZauuCWhL7v6tmfWqimBERGQL1bkKzsyuL7WYAewDzK2yiEREpFooTwuoTqnnxQRjQm9WTTgiIrKV6tgCCi9ArePuA5IUj4iIbKm6JSAzy3L3YjPbP5kBiYjI5qrjGNDXBOM9o83sXeB1YPWGje7+VhXHJiIiaaw8Y0A1gMXAIWy6HsgBJSAREamwshJQk7AC7gc2JZ4N0rRBKCISQ2n6jVtWAsoEarN54tkgTT8OEZGYqabXAc1z9zuSFomIiGxbmiagsu6Imp7zf4uIpJok3JLbzDLN7Dszey9cbm9mI8xsipm9uuFGpGaWGy5PCbe3K3WMm8P1k8zsiB29Z1kJqF/5QxcRkRR3DVD6Vjv3Ag+6+y7AUuDCcP2FwNJw/YPh6zCzLsCZBBNVHwk8EV5Lul3bTUDuvqSCJyEiIpXE2HRX1Io8yvUeZq2AY4CnwmUjqHx+I3zJc8CJ4fMTwmXC7f3C158AvOLu6919GjAF6FnW+5bVAhIRkTio+i64h4DfAYlwuSGwzN2Lw+XZQMvweUtgFkC4fXn4+o3rt7HPNikBiYjE2U60fsIWUCMzG1nq0b/04c3sWGChu49K9qn97NsxSNW47urh9Oo+h2XLa3DpVcdutu3kEyfQ/4JvOf3sU1ixsgannjSeg385HYDMzAStW63gjF+fQn699dz82y827tes2UpeeHkv/vPubsk8laTp3ncFl/55LpkZzof/bsBrjzWNOqRK1yR/FX84+zPq11kDbrwzfHdeHxrcouvUA3/g5APGkUgYX45vwxP/7Q3Arw/9jmN7TSThxoNv7c/XE1sDULvmem4643M6NF+KA3f/+5eMm94sqlPbKdk5Cf7vxTFk5ziZmc4Xgxrx0qNt2av3Ui787TQsA9atyeSBm3dl3sya7Nl9Of1v/on2nVdzzw27MWxg46hPIZkK3L17Gdv3B443s6MJJh6oCzwM5G+Ykg1oBcwJXz8HaA3MNrMsoB7BZAUb1m9Qep9timUCMrPbgVUEH8RQd/8k2oiq3seDO/Df9zoz4LovN1vfqNFq9t17HgsW5m1c98bbXXjj7S4A9Ooxm5NOmMiqVbmsWpXLFdceDUBGRoIX//U2Xw5vlbyTSKKMDOeKu+dw85kdKJiXzaMfTOargfWYOblG1KFVqpKE8eg7vflxdmPycgt5+oa3+GZSKxrUWcMBe07nvP87laKSTPJrrwWgXdOl9Os2hXPuOZ1G9Vbz8OXvc+ZdZ5DwDK496UtGTGzNrc8eTlZmCTVyinfw7vFVVGjcfH5X1q3JJDMrwX0vjWXk0PpceftP3HF5F2ZNzeOYX83lzMtm8uDNnVk4L5cHbu7MKRfMjjr0iqnCMmx3vxm4GcDM+gID3P1sM3sdOBV4BTgPeCfc5d1weXi4/VN393DKtpfN7AGgBdCJYEq37Yp1F5y7/7E6JB+AH8Y1ZeWqnK3WX3LhKJ56thv4tqvi+x40nSFD2221fu+uC5g3vzYLF9Wu7FBjoXO3NcydnsP8mbkUF2Uw5J18+hyxPOqwKt3iFbX4cXbwa33N+hxmLMincb3VnLj/eF4cvDdFJUGR0bJVNQE48BfTGfzdLhSVZDJvSV1mF9Rl97YLqVVjPXt1nMd/vwpaw8UlmaxamxvNSVUKY92a4NyzspzMrAQ4uENe7SCx1qpTwpKFwTkunFOD6T/WIpGq19MkoQx7G24ErjezKQRjPE+H658GGobrrwduAnD3ccBrwHjgI+AKdy8p6w1i0wIys1sIsupCgoGsUWb2LPCeu79hZvcAxxPck2iQuw8ws6bA34EO4WEuI7hZ3nvuvmd43AFAbXe/3cyuBi4NjzHe3c80s18SNDch+N91kLuvTMIp71DvXrNYvDiPadPrb3N7bk4x3feZx+P/6LHVtl9uJzGli4bNilg0d1PCLpiXzW77rIkwoqrXrAwMVgkAACAASURBVMFKOrVazLgZTbjihK/Yq8M8+h/zDYVFmTz2Tm8mzmpC43qr+WF6k437LFxWi8b11lBYlMWyVTW45awh7NJiMZNmNeaht/djXWF2hGe0czIynIff/I4Wbdby3sstmDS2Lg/f2ok/PTmOwnUZrFmVyXVn7B11mJUiWTMhuPsQYEj4fCrbqGJz93XAadvZ/y7grvK+XyxaQGa2L0H9+N7A0UCPLbY3BE4C9nD3rsCd4aZHgM/dfS+CmbvH7eCtbgK6hce4NFw3gCBT7w0cCKzdRnz9NwzgFRUn50suN6eYM08dx/Mvd93ua3r1nM24CY1ZtWrzX7JZWSX07jmH/w1rU9VhSpLUzCnirt8M4pG3+7BmfQ6ZGQnq5q2n/4Mn8vi7vfnz+Z9Q1s/dzAxn11YFvD2sC7+571TWFmbx636jk3cCVSCRMK46aR/O7duLXbuupG2n1Zx43hxu678H5/btxcdvNaP/TVOjDrNyRNMCqnKxSEAEX/xvu/sad19B0MdY2nJgHfC0mZ0MbMgChwB/A3D3EnffUR/MWOAlMzuHoBUEMAx4IGwd5ZcqO9zI3Z909+7u3j07K2/LzVWiefOVNGu6ir89/AHP/fM/NGq0hsce+pD6+Zvy4y8PnMGQoW232rf7vnOZ8lN9li2rmZRYo7B4fjaNWxRuXG7UvIiCean7a74smRkl3HXBIAaN6sTnY4PG/sJltfh8bHvAmDCzCe5Gfq11LFpei6b1N941hSb5q1m0PI+Fy2qxaHktxs8ICjWGjOnArq0KojidSrd6ZRZjR9Sj+4FL6bDbaiaNrQvA0A8bsXu3FRFHJ2WJSwIqU5gUehJc9HQsQf/i9hSz+XmVHpU+BnicoLX0TVjhcQ9wEVATGGZmsSgZmz6jPmeeeyrnXXwi5118IgUFeVx57VEsDZNKXl4hXfdcyPARrbfat++BM9K6+w1g0ug8WrYvpGnr9WRlJ+h7wjK+GlQv6rCqgHPzrz5nxoJ8Xh2yqTX8v+/bs0+nuQC0bryMrMwSlq2uwRc/tKVftylkZ5bQvMEKWjVazoQZTViyMo+FS2vTpskyAPbddQ7TF+RHckaVoW79QmrVCX4r5uSW0G2/ZcyaWpO8OsW0bBf8Pg3WJecHY5XamdZPzFtAcRkDGgo8a2Z/IYjpOOAfGzaaWW0gz90/MLNhwIZ29WCCcZ+HwikfagMLCG4l0ZCgku5Y4CMzywBau/tnZvYFQZdfbTNr6O7fA9+bWQ9gN2BiEs55MzcN+IKuey6gbt31vPDMW7z4764M/HiX7b5+/96zGPVdc9av3/x/YW5uMfvsPY9HnijzAuSUlygxHr+lJXe/PJWMTBj0SgNm/JheFXAAXdvP56gek5kytwHP/ja4KP0f7/XkvRGd+f2vhvDCja9RVJzJnS8fDBjT5jfg09Edeenm1yhJGA+8eQAJD36PPfjW/tx2zmCyshLMXVyXu1/uG92J7aQGjYu44Z5JZGQ6ZvC/jxrx9ZCGPPKHTtzyyAQSCWPViiwe+n0nADrtuZI/PDae2nWL6XXwEs65ciaXHbdvxGdRfuk6G7a5x+PMtihCmAl8C+wJvEfQTfYOQWvGgPvc/bmwCOFJgiKEEuAydx8edqddQ1CDPhWYTjAw9hlBzboBL7r7PWb2KHAwwRXA44Dz3X399uKsW6uF997jkko++9TkI3+IOoTYWHV676hDiI16H42POoTYGLj8mVE7uAZnh2o2a+0df319hfcfd9/1Ox1DVYlLC6g81RPbqsZYQDD/0JbrHyEoUNjSAdt47VU/I0wRkaRL1xZQSowBiYhI+olNC0hERLYjTVtASkAiInGWAtVsFaUEJCISY0b63p5aY0AiIhIJtYBEROJOXXAiIhKFdC3DVgISEYk7JSAREYlEmiYgFSGIiEgk1AISEYkz1xiQiIhERQlIRESioBaQiIhEI00TkIoQREQkEmoBiYjEnLrgREQk+TQbtoiIRCZNE5DGgEREJBJqAYmIxJihMSAREYmKEpCIiETBPD0zkBKQiEicpXEVnIoQREQkEmoBiYjEnIoQREQkGkpAIiIShXRtAWkMSEREIqEW0M+0vkEmU86sHXUYsdBxlEUdQmws66jfchsMmzg06hBiI7N5JR0oTVtASkAiInGmW3KLiEhklIBERCTZ0nkuOHVci4hIJNQCEhGJO80FJyIiUUjXLjglIBGROEvjyUiVgEREYs4SUUdQNVSEICIikVALSEQk7tQFJyIiUVARgoiIJJ+TtmXYGgMSEZFIqAUkIhJz6oITEZFoKAGJiEiypfNkpEpAIiJx5q4iBBERkcqkFpCISMypC05ERKKhBCQiIlFQC0hERJLPgUR6ZiAVIYiISCTUAhIRibv0bACpBSQiEnfmFX/s8Nhmrc3sMzMbb2bjzOyacH0DM/vYzCaH/60frjcze8TMppjZWDPbp9SxzgtfP9nMztvReysBiYjE3YaLUSvy2LFi4AZ37wL0Bq4wsy7ATcBgd+8EDA6XAY4COoWP/sDfIEhYwG1AL6AncNuGpLU9SkAiItWYu89z92/D5yuBCUBL4ATgufBlzwEnhs9PAJ73wFdAvpk1B44APnb3Je6+FPgYOLKs99YYkIhIzO1kGXYjMxtZavlJd39ym+9j1g7oBowAmrr7vHDTfKBp+LwlMKvUbrPDddtbv11KQDGTYQn+c8RbzF9bi/6fH8W/D32HWtlFADTMXcvYxU247H9HcHy7yfTffTRmsLoomz9+cyATlzUE4C+9hnBIyxksXleToz84PcrTqVTZuQnuf3MK2bkJMjPhf+/X44X7m7PX/iu5+A9zyc52Jn9fkwduaEOixKIOt1LkZBbz/CnvkJNZQqYlGPRTBx4f0ZNerWYzYP/hZGeWMH5hY/4w+GBKfFOHxp5NFvLSaW/x248OY9BPHQEYe8Xfmby4AQDzVtbmyvePjuScdsbbTzXiw5ca4g5Hnb2Eky9eBMA7Tzfi3WcbkZHp9Oq3gov+MG/jPgtnZ3Nx390454b5nHbZoo3rS0rgqiN3pWHzIv78/LSkn0u5OTtbhFDg7t139CIzqw28CVzr7ivMNv0bcnc3q/yrkVIuAZlZX2CAux9bxmv2Blq4+wc/89hDwmOP3NFrq8r5nX9gyor61M4uBOBXn5ywcdtjBwzikzntAJi1qg5nfXI8K4pyOaj5TO7sOZRTB50EwFtTd+XFH/fgr30+S3r8ValovfG70zuybk0mmVnOA29PZtTndfntQzO58YyOzJlag3MHzOOw05Yw8JWGUYdbKQpLMrng7eNZU5RNVkYJL5zyH4bNaMNdh37Khf85nhnL8rmy19ecsPsk3hq/OxD8iLl+v+F8ObP1ZsdaX5zJKa+k7g+S6RNr8OFLDXnk/R/JznF+f1ZHeh26nEVzc/hyYD3+9skkcnKdZQWbf639408t6XHIyq2O95+nGtO603rWrIr3SEQwG3bVlsGZWTZB8nnJ3d8KVy8ws+buPi/sYlsYrp8DlP7jahWumwP03WL9kLLeN5afvJntbGLcG0i5n3fNaq6ib4sZvPbTblttq51VSJ9mc/hkVjsAvitoxoqiXABGFzSlWd6qja/9ZlELlhXWSErMyWWsW5MJQFaWk5ntlJRAUaExZ2pwvt8OrcMBRy+LMshKZqwpygYgKyNBVkaCEjeKEpnMWJYPwJezWnFYx6kb9zi76/d8/FNHlqytGUnEVWXm5Fx267aGGnlOZhZ07bOKYR/k897zDTnjygXk5AZf0vmNijfu8+WH9WjWupC2u67b7FiL5mbz9eC6HHXW4qSeQ4UlduKxAxY0dZ4GJrj7A6U2vQtsqGQ7D3in1Ppzw2q43sDysKtuIHC4mdUPiw8OD9dtV5UlIDNrZ2Y/lFoeYGa3m9kQM7vXzL42sx/N7MBw+/lm9q6ZfQoMNrNaZvZM+LrvzOyEbbxHTzMbHm7/0sw6m1kOcAdwhpmNNrMztncsM6tpZq+Y2QQzexuI9F/srft+yb3f9cZ96+6jQ1tPZ/j8lqwqztlq22kdJzJ0bptkhBi5jAzniUETeXXsD3w3tA6TvssjM8vp1HUNAAccs4zGLYoijrJyZViCN898jf9d+CzDZ7Xi+wVNyLIEezQJfpAe3nEqzWoHP0Ca1FpFv47TeOX7PbY6Tk5WCa+e/gYvn/Ymh3SIcZfTdrTbbR0/fF2LFUsyWbfG+ObTuiyam82cn2rww4jaXH1MJwacvAuTRgf/jNeuzuC1J5pwzg3ztzrW329ryUW3zsVi+RM86fYHfg0cEn5njjazo4F7gMPMbDJwaLgM8AEwFZgC/BO4HMDdlwB/Br4JH3eE67Yrqi64LHfvGZ7kbQQnB7AP0NXdl5jZ3cCn7n6BmeUDX5vZJ1scZyJwoLsXm9mhwN3ufoqZ/RHo7u5XApRxrEuANe6+u5l1Bb7dVrBm1p+g3JDM+mVWFVbYwS2CMZtxSxvTq8ncrbYf13bKNltGvZvM4bSOEznz463yc1pKJIzLD9+NWnWLue3p6bTtvI6/XN6OS2+fQ3aOM2poHRLl+NWXShKewSmvnE6dnPU8csxH7NJgCQMGHsaNBw4jJzPBlzNbkQh/tNx04DAeGNYbZ+sfMYc9ew4LV9emVd0VPHPSu0wuaMCsFfWSfToV1qbTek6/fCE3/6ojNfISdNhjLRmZwVjOymWZPPzeZCaNzuOuS9rx3FcTeOG+Zpx08SJq1tr8D+Krj+uS36iYTl3XMubL2hGdzc9TlV1w7v4FbOMPJtBvG6934IrtHOsZ4JnyvndUCWhDH+MooF2p9R+XypiHA8eb2YBwuQaw5c/8esBzZtaJYJguezvvt71jHQQ8AuDuY81s7LZ2DitGngTIbd26Sv4S9m08n36tZvDLFjPJzSyhdnYR9/cZzA3D+1E/dy1dGy7ksqGHb7ZP5/zF3N1rKBcMOSpNu9y2b/WKLMYMq02Pvit54x9NuOHkTgDsc9AKWnVYH3F0VWNlYS5fz27JAW1n8ex3e3Pum8GY336tZ9E2fzkAezRZxH1HBr/T6tdYy4FtZ1DsGXw6tT0LVwdftrNX1OWbOS3YvXFBSiUggCPPWsKRZwVfEc/8pTmNmxcya0ou+x+9HDPYrdsaMjJg+ZJMJn6Xxxfv5/P0nS1YtSITy3Bycp3F87P5alBdvhnchcL1xpqVmdx7ZRtufGxmxGe3HTtfhBBbVZmAitm8i6/0N+SGb4iSLWJYXeq5Aae4+6TSBzWzpqUW/wx85u4nheWDQ7YTy/aOVfYZJNF9Y3px35heAPRqMpcLdx/DDcODHx9Htp7GZ3PaUpjY9FE1z1vJEwcO4obhBzN9ZX4kMSdbvQbFFBcHySenRoJ9DlrJa080oV7DIpYvziY7J8HpVyzk34803fHBUkT9GmspTmSwsjCX3Mxi+rSZxdOjutGg5hqWrM0jO6OEC/f9jidHBhejH/H8ORv3vevQT/l8Wls+ndqeurnrWVuURVEik/waa+nWfD7PjNo7qtOqsGUFWeQ3Kmbh7GyGfVCPh9+bjGXAmGG12Xv/Vcz+KZeiQqNegxIe+M+Ujfu9cF8zatQq4YQLCgC44PdBldyYL2vzxt8bxzf5AJC+d0StygS0AGhiZg2BVcCxwEc/Y/+BwFVmdlVYAtjN3b/b4jX1CCovAM4vtX4lUKccxxoKnAV8amZ7Al1/RnxJc2zbKfxj/OZfFlft+S35uev4U48vAChJGCcNPAWAB/f7hF5N51E/dx1fnPgiD4/tzutTt+6+SzUNmhYx4KGZZGQ4GRkw9L/5jPikHhfdOodeh67AMuD95xsyZlidHR8sRTSutYa7D/uUDEuQYc7Aybvw+fR23LD/l/yy3QwyzHn1+z0YMbtVmcfpUH8ptx38OY5hOE+N6sZPSxsk6Swqzx0XtWPl0iwys50r755N7XolHHHmEh64vjX9D+5Mdrbz24dnEqPflpUiXW/HYF6FmdXMrgauIUgSU4HpBGV6A9x9pJk1Aka6ezszO5/Nx21qAg8B+xG0pKa5+7Gly7DNrA/BFbqrgfeBc8JjNSBIOtnAXwiqNrZ1rJrAv4C92HT17xVllWHntm7tLa+7trI+opTW8bcjog4hNmbf1CfqEGJj3FVPRB1CbGQ2nzKqPNfglKVunZbec59tDrmUy+Cht+x0DFWlSseA3P0RwjGW7WwvIBwDcvdngWdLbVtLUCSw5T5DCLva3H04sGupzbeG65cAPbbYdVvHWgucucMTERGJkrrgREQk6RwszSo7N1ACEhGJuzRtAekyLBERiYRaQCIicZeeDSAlIBGRuKvqyUijogQkIhJ3SkAiIpJ0TrlmtU5FKkIQEZFIqAUkIhJjhmsMSEREIqIEJCIikVACEhGRpFMRgoiISOVSC0hEJOZUhCAiItFQAhIRkeRL31tyawxIREQioRaQiEicOWnbAlICEhGJuzQtw1YCEhGJOVXBiYhINNI0AakIQUREIqEWkIhInDmQSM8WkBKQiEispe91QEpAIiJxpwQkIiKRSNMEpCIEERGJhFpAIiJxlsZFCOZp2rSrKma2CJgRdRxAI6Ag6iBiQp/FJvosNonDZ9HW3RvvzAHq5Tb1/VqcXeH9P5r+4Ch3774zMVQVtYB+pp39Y6osZjYyrn9UyabPYhN9Fpuk1WeRpg0FjQGJiEgk1AISEYmzNB4DUgJKXU9GHUCM6LPYRJ/FJunzWaRpF5wSUIpy9/T5x7WT9Flsos9ik7T6LJSAREQk+dJ3Kh4VIYiISCTUAhIRiTMHEul5S1QloBRjZnnuvibqOKJkZgacDXRw9zvMrA3QzN2/jji0pDOzWsBad0+Y2a7AbsCH7l4UcWhJZWb7lLXd3b9NVixVIk274JSAUoSZ7Qc8BdQG2pjZXsAl7n55tJFF4gkgARwC3AGsBN4EekQZVESGAgeaWX1gEPANcAZBgq5O7g//WwPoDowBDOgKjAT6RBRX5UjTBKQxoNTxIHAEsBjA3ccAB0UaUXR6ufsVwDoAd18K5EQbUmQsbBGfDDzh7qcBe0QcU9K5+8HufjAwD9jH3bu7+75AN2BOtNHtLA+uA6roI8aUgFKIu8/aYlVJJIFEr8jMMgl6xzGzxgQtourIzKwPQYvn/XBdZoTxRK2zu3+/YcHdfwB2jzAeKYO64FLHrLAbzs0sG7gGmBBxTFF5BHgbaGpmdwGnArdGG1JkrgVuBt5293Fm1gH4LOKYojTWzJ4CXgyXzwbGRhjPznNwT8/fV5oNO0WYWSPgYeBQgr7tQcA17r440sAiYma7Af0IPovB7l5dk/FGZpYB1Hb3FVHHEhUzqwFcxqbu6aHA39x9XXRR7Zx6WY29T90TK7z/wKVPaTZs2TnuXkD1G1guSyNgjbv/y8wam1l7d58WdVDJZmYvA5cSdMd+A9Q1s4fd/a/RRhYNd19nZn8HPnD3SVHHU2nStKGgMaAUYWa7mtlgM/shXO5qZtWy28nMbgNuJOh6AshmU5dLddMlbPGcCHwItAd+HW1I0TGz44HRwEfh8t5m9m60Ucn2KAGljn8SfOEWAbj7WODMSCOKzknA8cBqAHefC9SJNKLoZIdjgicC74bX/6Tnz+XyuQ3oCSwDcPfRBEk5dbkHF6JW9BFjSkCpI28bF1oWRxJJ9Ao9GLzcUAVXK+J4ovQPYDpQCxhqZm2BajsGBBS5+/It1qV+Qnav+CPGNAaUOgrMrCObvnRPJbjmoTp6zcz+AeSb2cXABQQtxGrH3R8hqArcYIaZHRxVPDEwzszOAjLNrBNwNfBlxDHtNI95S6ailIBSxxUE9zfZzczmANOohkUJ4TQ8rxJMObMC6Az80d0/jjSwiJjZH7ez6Y6kBhIfVwG3AOuBl4GBwJ8jjWinxb8lU1FKQCkgvOjycnc/NOxuynD3lVHHFQV3dzP7wN1/AVTLpLOF1aWe1wCOpfpeHwZwjLvfQpCEADCz04DXowtJtkcJKAW4e4mZHRA+X72j11cD35pZD3f/JupAoubu95deNrP7CH71V1c3s3Wy2da61KFbcksMfBeWk75OqV+97v5WdCFFphdwtpnNIPgsjKBx1DXasGIhD2gVdRDJZmZHAUcDLc2s9JhYXdKhWCdNZ0JQAkodNQgmIj2k1DoHqmMCOiLqAOLCzL5nU5VXJtCY6jn+M5dg1uvjgVGl1q8EroskokrigKsFJFFy999EHUOMbGv8q1qOiRGM+WxQDCxw99T/xf8zhbPDjzGzl6vbvZAqg5kdSTDVVybwlLvfk4z3VQJKEVt0K2ywHBjp7u8kO56IfQu0BpYSdL/lA/PNbAFwsbuPKmvnNNMcGLehKMXM6phZF3cfEXFcUWlnZn8BuhD0GgDg7h2iC2knuVdpF1xY5PQ4cBgwG/jGzN519/FV9qYhXYiaOmoAewOTw0dXgr7+C83soSgDi8DHwNHu3sjdGwJHAe8BlxPcrK46+RuwqtTy6nBddfUvgvMvBg4GnicNpmnyhFf4UQ49gSnuPtXdC4FXgBOq9IRCSkCpoytwsLs/6u6PEsyKvRvBtDSHRxpZ8vV2942VXu4+COjj7l8BudGFFQnzUlPaezBvf3Xu2ajp7oMJPpcZ7n47cEzEMe08T1T8sWMtgdL3Gpsdrqty1fkPNdXUJ7gd94ZpRmoBDcIS7fXRhRWJeWZ2I8EvNQhuQb0g7EpIz3Kh7ZtqZlezqdVzOTA1wniitj68LcVkM7uS4G6otSOOaaesZOnAT/yNRjtxiBpmNrLU8pPu/uTOxlUZlIBSx/8Bo81sCMG4x0HA3eGFqZ9EGVgEziKYdPI/4fKwcF0mcHpUQUXkUoKpeG4lKJgaDPSPNKJoXUNQin41wQwIhwDnRRrRTnL3I6v4LeYQjKlu0Iok3cZcN6RLIWbWgmCq/QkEv+pmu/vQaKMSkVRmZlnAjwQ3eJxDcF+ps9x9XFW/t1pAKcLMLiL4ddeK4H4nvYHhbH5dULVgZo2B3wF7sHmlU3X8LP4PuBNYS3APnK7Ade6e8gPvP4eZ/ZcyZr129+OTGE5KcffisLtyIEEvwjPJSD6gFlDKCC847AF85e57h7ekvtvdT444tKQzs0EEE5IOIOiCOg9Y5O43RhpYBMxsdPj3cBLBNUHXA0Pdfa+IQ0sqM/tlWdvd/fNkxSLlpxZQ6lgX3m4YM8t194lm1jnqoCLS0N2fNrNrwi+Wz82sus4Lt+Hf8DHA6+6+PJgwvHopnWDMrCbQJq1uyZ2mVIadOmabWT7BwPvHZvYOMCPimKKy4Ur3eWZ2jJl1AxpEGVCE3jOzicC+wOCwe3JdxDFFxsyOQ7fkThnqgktBYXdDPeCj8MKxasXMjgX+R1C58yjBhJN/cvdq+UVjZg2A5WFJfh5Q193nRx1XFMxsFMG46BB37xau+z68fYfEjLrgUlB178929/fCp8sJrnavdszsEHf/1MxOLrWu9Euq4yS1EN6Se4vPQr+yY0pdcJJyzOz/zKyumWWb2WAzW2Rm50QdV5IdFP73OILigy3/W11tdktuM3uUNLgld7pSApJUdLi7ryD4op0O7AL8NtKIkm+lmV0P/FDqMQ74PnxeXV1FUJ6/4Zbcy4FrI41ItktdcJKKVPm1aXqZzgTl+e8QzJBxHPB1VEFFKZyK6X13P5hSt+SW+FICklS0ofJrLXBZdaz8cvc/AZjZUGCfUrdjuB14P8LQIhMWYSTMrJ67L9/xHhI1VcFJSlLlV8DMJgFd3X19uJwLjHX3anmNWHh5QjeCW3aUvnX91ZEFJdulFpCkDFV+bdPzwNdm9na4fCLwbHThRO4tquffQUpSC0hShpnd7u63m9m/CEprrfR/3f2CSAOMiJntAxwYLg519++ijEekvJSAJGWY2Q1snXgIn+PuD0QUmsSEme0P3A60Jejh2fDjJHVvyZ3G1AUnqUSVX7IjTwPXAaOAkohjkR1QC0hSTlj5dUypyq86BOW3B5W9p6Q7Mxvh7r2ijkPKRy0gSUVNgdJz4BWG60Q+M7O/EhQibLxVvbt/G11Isj1KQJKKVPkl27Oh9bNv+N8N44XV7maFqUAJSFKOu99lZh+yqfLrN6r8ktCQbazTOENMKQFJSgq7VNStIltaVep5DYL5AidEFIvsgIoQRCRthTNDDHT3vlHHIlvTbNgiks7ygFZRByHbpi44EUkbZvY9m8Z8MoHGwB3RRSRlUReciKQNM2tbarEYWODuxVHFI2VTAhIRkUhoDEhERCKhBCQiIpFQApKUZGYlZjbazH4ws9fDm9JV9FjPmtmp4fOnzKxLGa/ta2b7VeA9pptZo/Ku3+I1q8ravo3X325mA35ujCLJpgQkqWqtu+/t7nsSzAV3aemNZlahCk93v8jdx5fxkr7Az05AIrI1JSBJB/8DdglbJ/8zs3eB8WaWaWZ/NbNvzGysmV0CYIHHzGySmX0CNNlwIDMbYmbdw+dHmtm3ZjbGzAabWTuCRHdd2Po60Mwam9mb4Xt8E96PBjP7//bu3bWKIAzj8O8Niki8YMTCQkFEiSLe8JoiqIgQLSSCCNoZCAom4D+gaGUhpBMvwUJEEfECIiQBRRJF8YaC0SJFwMJGkniL2uhnsXPM4RDJKYKbE96n2vPtzs7sFudjZnZn50rqktQrqZ2Rbxf9k6Tbkl6kMs0l+9pS/J6keSm2WFJHKtMjqXY8bqbZ/+L3gKyipZ5OA9CRQmuBFRHRn/7EP0fE+vRG/CNJXcAasm8KLSdbRfstcLHkvPOAC0B9OldNRAxKOgt8i4jT6bgrQFtEPJS0EOgElgHHgYcRcVLSLqCpjMs5mOqYDjyTdCMiBoBq4HlEHJV0LJ37CHAeOBQRfZI2AmfwoptWQZyArFJNl/QqbfeQfYisDngaEf0pvgNYWZjfAWYDS4B64GpE/AI+SLo/yvk3kX3euh8gIgb/0Y7twHLpbwdnlqQZ9pSYEAAAAWFJREFUqY49qexdSUNlXFOrpMa0vSC1dQD4DVxL8cvAzVRHHXC9qO5pZdRhNmE4AVml+hERq4sD6Y94uDgEtEREZ8lxO8exHVXApoj4OUpbyiZpC1ky2xwR3yU9IFtMczSR6v1Ueg/MKonngGwy6wQOS5oKIGmppGqgG9iX5ojmA1tHKfsEqJe0KJWtSfGvwMyi47qAlsIPSYWE0A3sT7EGYM4YbZ0NDKXkU0vWAyuoAgq9uP1kQ3tfgH5Je1MdkrRqjDrMJhQnIJvM2snmd15KegOcI+v13wL60r5LwOPSghHxEWgmG+56zcgQ2B2gsfAQAtAKrEsPObxl5Gm8E2QJrJdsKO79GG3tAKZIegecIkuABcPAhnQN2xhZ2+wA0JTa1wvsLuOemE0YXorHzMxy4R6QmZnlwgnIzMxy4QRkZma5cAIyM7NcOAGZmVkunIDMzCwXTkBmZpYLJyAzM8vFH1IG7ARmolBKAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x432 with 2 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "###########   TEST\n",
        "\n",
        "test_annot = pd.read_csv(\"competition_test_stances.csv\")\n",
        "unrelated_num = test_annot.Stance.value_counts().unrelated\n",
        "unrelated_num = sum(map(lambda x : x == 'unrelated', gold_labels))\n",
        "max_score = len(test_annot) - (unrelated_num * 0.75)\n",
        "print(f'Score test: {score_test}/{max_score}, so {(score_test*100)/max_score}%')\n",
        "print()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zL2u2JLxY6hE",
        "outputId": "6efc2437-29ca-4dcb-eee1-77bccda32732"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Score test: 6996.25/11651.25, so 60.047205235489756%\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "f1_summary = f1_score(gold_labels, pred_labels, average=None)\n",
        "f1_micro = f1_score(gold_labels, pred_labels, average=\"micro\")\n",
        "f1_macro = f1_score(gold_labels, pred_labels, average=\"macro\")\n",
        "print(f1_summary)\n",
        "print(f1_micro)\n",
        "print(f1_macro)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6FL-pu-m-kEr",
        "outputId": "dc658869-14c2-43c3-d053-be59007b7233"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[0.26899053 0.         0.44834409 0.67584709]\n",
            "0.5345689214181718\n",
            "0.3482954255965092\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "MODEL_TO_LOAD = MODEL_DIR + \"cnn_model/seq_length_5.h5\"   #CNN experiments/cnn_net_3_layers_2048.h5\" #  cnn_net_3_layers_2048 cnn_net_window_5\n",
        "model_loaded = tf.keras.models.load_model((MODEL_TO_LOAD), custom_objects={'KerasLayer':hub.KerasLayer, 'f1_training':f1_training, 'recall_training':recall_training, 'precision_training':precision_training,})"
      ],
      "metadata": {
        "id": "vhNs5rQT8nfh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "###########   TRAINING\n",
        "\n",
        "# compute score on validation\n",
        "headline_train, body_train, labels_train, headline_val, body_val, labels_val = get_dataset_splitted()\n",
        "\n",
        "BERT = False\n",
        "\n",
        "if BERT:\n",
        "  start_time = time.time()\n",
        "  pred_labels = get_predictions(headline_train, body_train, model_loaded)\n",
        "  inference_time = time.time() - start_time\n",
        "else:\n",
        "  vectorizer = get_vectorizer(SEQ_LENGTH)\n",
        "  vect_head_train = vectorizer(np.array([[s] for s in headline_train])).numpy()\n",
        "  vect_body_train = vectorizer(np.array([[s] for s in body_train])).numpy()\n",
        "\n",
        "  start_time = time.time()\n",
        "  pred_labels = get_predictions(vect_head_train, vect_body_train, model_loaded)\n",
        "  inference_time = time.time()\n",
        "\n",
        "gold_labels = np.array([LABELS[i] for i in labels_train])\n",
        "print(f'Inference time: {inference_time}')\n",
        "\n",
        "\n",
        "fig, ax = plt.subplots(figsize=(6, 6))\n",
        "ConfusionMatrixDisplay.from_predictions(gold_labels, pred_labels, xticks_rotation='vertical', ax=ax)\n",
        "score_test = score_metrics(gold_labels, pred_labels)\n",
        "print(get_confusion_matrix(gold_labels, pred_labels))\n",
        "\n",
        "\n",
        "unrelated_num = sum(map(lambda x : x == 3, labels_train))\n",
        "max_score = len(labels_train) - (unrelated_num * 0.75)\n",
        "print(f'Score test: {score_test}/{max_score}, so {(score_test*100)/max_score}%')\n",
        "print()\n",
        "\n",
        "f1_summary = f1_score(gold_labels, pred_labels, average=None)\n",
        "f1_micro = f1_score(gold_labels, pred_labels, average=\"micro\")\n",
        "f1_macro = f1_score(gold_labels, pred_labels, average=\"macro\")\n",
        "print(f1_summary)\n",
        "print(f1_micro)\n",
        "print(f1_macro)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 525
        },
        "id": "PwG1Nmeg_tYK",
        "outputId": "c47c7d15-978b-4ce7-f92c-20d6530417c9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Inference time: 1643631176.1254628\n",
            "[[2507, 72, 165, 198], [401, 167, 39, 65], [31, 10, 6735, 351], [320, 5, 646, 28265]]\n",
            "Score test: 16654.75/18050.0, so 92.27008310249307%\n",
            "\n",
            "[0.80857926 0.36069114 0.91557912 0.97272649]\n",
            "0.9423918753283138\n",
            "0.7643940039458395\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaYAAAF8CAYAAACE1B9KAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3xV9f3H8dcnC0KAQNhTQBEcRUTAbd2oP1ytq9o6qpWqOOqq1las2ypaqXuPOnBWRRQRB04EFBVUFNmbMMIII8n9/P44B7hgCCEhOedy38/H4zy493vG/ZxLks/5jvM95u6IiIjERUbUAYiIiCRTYhIRkVhRYhIRkVhRYhIRkVhRYhIRkVhRYhIRkVjJijoAERHZtD4H5fnCRWVV3n/sN6uHufsRWzGkGqfEJCISY4WLyhg1rG2V989u9XPTrRhOrVBTnoiIxIpqTCIiseaUeSLqIGqVEpOISIw5kCC9po5TYhIRibkE6VVjUh+TiIjEimpMIiIx5jhlafYUCCUmEZGYUx+TiIjEhgNlSkwiIhIn6VZj0uAHERGJFdWYRERizEGDH0REJF7S6y4mJSYRkVhzXIMfREQkRhzK0isvafCDiIjEi2pMIiIxFkziml6UmEREYs0ow6IOolYpMYmIxJgDCfUxiYiIREc1JhGRmFNTnoiIxEYwiasSk4iIxEjClZhERCQm0rHGpMEPIiISK6oxiYjEmGOUpVkdQolJRCTm1MckIiKxkY59TEpMWygnM9dzsxpGHUYs+JqSqEMQibVlLC5092bVO4pR5mrKkwrkZjVkn9anRR1GLJROmxF1CPFh6XVFK5XzbuLFaVHHkIqUmEREYiyYXVw1JhERiRH1MYmISGy4p18fU3qdrYiIbMDM2pnZ+2b2nZlNMLOLw/LrzGyWmY0Ll6OS9rnazCaZ2UQz65NUfkRYNsnMrkoq72hmo8LywWaWU1FMSkwiIjGXwKq8VEIpcJm77wzsBVxgZjuH6+5y9+7hMhQgXHcKsAtwBHCfmWWaWSZwL3AksDPwu6Tj3BYeawdgMXB2RQEpMYmIxFhwH1NGlZfNHt99jrt/Gb5eBnwPtKlgl2OB5919tbtPASYBvcNlkrtPdvc1wPPAsWZmwMHAS+H+TwLHVRSTEpOISKwFfUxVXYCmZjYmaTl3k59k1gHYHRgVFvU3s2/M7DEzaxyWtQGS7xWZGZZtqrwJsMTdSzcq3yQNfhARibGtMFy80N17bm4jM6sPvAxc4u5Lzex+4IYwhBuAgcAfqxNIZSkxiYikOTPLJkhKz7j7KwDuPi9p/cPAkPDtLKBd0u5twzI2Ub4QaGRmWWGtKXn7cqkpT0Qk5srcqrxsTtgH9CjwvbvfmVTeKmmz44Hx4evXgVPMrI6ZdQQ6A18Ao4HO4Qi8HIIBEq+7uwPvAyeE+58BvFZRTKoxiYjEWC089mJf4A/At2Y2Liz7G8Gouu4ETXlTgX4A7j7BzF4AviMY0XeBu5cBmFl/YBiQCTzm7hPC4/0VeN7MbgS+IkiEm6TEJCISc4kavMHW3T+GcseVD61gn5uAm8opH1refu4+mWDUXqUoMYmIxNja4eLpJL3OVkREYk81JhGRGHMqN4hhW6LEJCISc3rshYiIxIY7ml1cREQkSqoxiYjEWqVnCd9mKDGJiMSYk35NeUpMIiIxl273MSkxiYjEmGMk0my4eHqlYRERiT3VmEREYk5NeSIiEhtOzU7iGkdKTCIisWaUpdlw8fRKwyIiEnuqMcVE0+YruezacTQqWI07vP1ae15/oROnnj2RPsdOZ+niOgA8+UAXxnzWAoATT5/E4UdPJ1FmPHjXLnw5qjlt2i/nqhu+XHfclm2K+e/DO/La4E6RnNfW1Hb7VfztgWnr3rdsv4anb29Jk1Yl7HXYUkrWGHOm5TDwL+1ZsTQzwkhrxqUDp7PnoUtZUphFv0O6ris/5qwFHHNmIYkyY9SIhjx6U2tatF3Nwx/8wMzJwc/ND1/mMeiqdps6dMop77votPNKLrx1Brn1EsybmcNt/bejeHkmmVnOX+6Yzg67riQzy3n3pQIG39Mi4jOoPDXlSWTKyoxHBu3Mzz/mk1uvlLsf/4ivvmgGwGvPd+KVZ7ffYPt2HZZxwKGzOO/UX9Ok6WpuGvQ55558ELOm1+fCMw4AICPDeer1d/n0w5a1fj41YebPdTn/sC5AcG7PfPkdn7yVT9sdVvPYza1IlBlnXzObUy6cx6M3tY442q3vnRcKeP3xplxx9/R1Zbvts4x9+hRx3mFdKFmTQX6TknXr5kyrw/mHdy3vUCmvvO/iktun8/ANbfj28/ocfvJCTjhvPk/d3ooD+i4hO8f586FdqVM3wUMffM8H/2vEvJl1IjyDLaOmvG2MmaVE8l28sC4//5gPwMriLGZMrU+TZqs2uf1eB8xj5LttKC3JZN6cesyemceOOy/ZYJvdehYyZ1Y9FsytV6OxR6H7/suZMy2H+bNy+PLDBiTKgl/c78fm0bRVyWb2Tk3jR9Vn2ZINa4J9T1/I4HtbULIm+FUuWpgdRWi1rrzvom2n1Xz7eR4AX33UgP2OCn4f3KFuvQQZmU5OboLSkgyKl6dOjdrdSHhGlZdUlDJRm9n/zGysmU0ws3PDsrPN7Ecz+8LMHjaze8LyJ8zsATMbBfzLzLY3s7fD/T8ys67hds3M7GUzGx0u+0Z4ius0b1lMpx2LmDihEQB9T5jKPU9/yMXXfE39BmsAaNJsJYXz6q7bZ+GCujRptnKD4xxw2Gw+HL7t1RwADjx2MR/8r/Evyvv8bhGj32sYQUTRaNNpFbv2Xs7db/zI7S/9xI67Fa9b17L9Gu4dNpHbX/qJXXsvjzDK2jHtx7rs3acIgP37LqFZ6+AC5aM3G7GqOIPnvhrPf7/4jpceaMayJSlxvbpOmWdUeUlFqRT1H919D6AncJGZtQH+AewF7Ats3GbRFtjH3S8FHgIuDPe/HLgv3OZu4C537wX8FnikvA82s3PNbIyZjVlTtrK8TbaaurmlXHPLWB7+9y6sLM5m6CsdOOeEg7nw9ANYXFiHsy/6vlLHycpKsOd+c/l4xLaXmLKyE+x1+FJGvpG/QfnvLppHWSm890qjiCKrfZmZ0KBRGRcf3ZlHbmzNNQ9MBZxF87P5fe+duaBPFx78Zxuuunca9eqXRR1ujbrz0vYcfcZC7nlrIrl5CUpLglp0l+4rSJQZp/bYldP32onf9ltAy/arI45WKpJKlw0Xmdnx4et2wB+AD919EYCZvQjsmLT9i+5eZmb1gX2AF83WtdOubVw+FNg5qbyhmdV39w0uL939IYLkRn6dFr51T2u9zMwEf7t5LO8Pa8OnH7YCYMni9e3gb7/WngF3jAZg4YJcmrZY39TXpNkqFi7IXfe+597z+Xli/gb7byt6HbyMSd/msqRwfbPVYSctovehS7nq5O0hjdrjC+dk88lb+YAxcVweiQTkF5RRtChrXfPepG/rMXtqDm06reanb7a9Zt21Zvxcl7+dGvTFtum0ij0PWQrAQccvYcwHDSgrNYoWZvPd6Dx23K2YudNT43fDIe1mF0+JGpOZHUiQRPZ2992Ar4AfNrPbivDfDGCJu3dPWnZKWrdXUnmbjZNS7XEuvuZrZkyrz/+eXz+CrnGT9clnnwPnMm1yAwBGfdSCAw6dRVZ2GS1aFdOm3Qp+/G59TSFoxmtTe+HXogOPW7JBM17PA5dy4vnzue7MjqxemRI/0lvNp8Py2W2f4Ee2TadVZOc4RYsyyS8oJSMjuIZq2X41bTquYe70nChDrXFrB36YOadePI8hTzcBYMGsbLrvG3xHdXLL6NpjBTMm1d3kceLH0q4pL1VqTPnAYncvDvuH9gLygF+bWWNgGUFT3Lcb7+juS81sipmd6O4vWlA96ubuXwPvABcCtwOYWXd3H1dL57SBnbst5pAjZzFlUgP+8+RIIBga/uvDZtNpx6W4w/w59fjPbb8CYPqUBnw8ojUPPPshZWXGfXfsSiIRXFXVqVvK7r0XcE+47bakTm4ZPfZfxt1Xtl1XdsFNs8iu49wy+GcAfhibx6Cr2m7qECnrqnun0m3v5eQXlPLfMRN4+o6WDHu+gEsHzuDBET9QUmLcfkl7wPjVXss5/fK5lJZCImEMurptyvWrVKS87yI3L8HRZxYC8MnQfN4ZXADA60805bK7pvPQez+AOe8MbsKU73MrOnysBMPF06vGZO411jK11ZhZHeB/QAdgItAIuI6g6e4KYBFBDWqmu19jZk8AQ9z9pXD/jsD9QCsgG3je3a83s6bAvcBOBEl6pLv/uaJY8uu08H1an7a1TzEllU6bEXUI8WHp9YdDKufdxItj3b1ndY7RapfGfsZzh1R5/9t2e7naMdS2lLiEcvfVwJEbl5vZGHd/KBwS/ipB8sLdz9xo/ynAEeUctxA4uSZiFhGRqkmJxFSB68zsUKAuQbPc/yKOR0Rkq0rH5zGldGJy98ujjkFEpKYlUmOc2laT0olJRGRb5w5laVZjSq80LCIisacak4hIzKmPSUREYiMY/JBejVtKTCIiMZduj71QYhIRibF0nPkhveqHIiISe6oxiYjEmvqYREQkZtLtsRdKTCIiMZaON9gqMYmIxFy6NeWl19mKiEjsqcYkIhJjml1cRERiR4MfREQkNnSDrYiISMRUYxIRibl0G5WnxCQiEmeuwQ8iIhIjjgY/iIhIzKRbjSm9Gi5FRCT2VGMSEYkxDRcXEZHYSYQDIKqybI6ZtTOz983sOzObYGYXh+UFZjbczH4K/20clpuZDTKzSWb2jZn1SDrWGeH2P5nZGUnle5jZt+E+g8yswsCUmEREYmztlEQ1lZiAUuAyd98Z2Au4wMx2Bq4CRrh7Z2BE+B7gSKBzuJwL3A9BIgMGAHsCvYEBa5NZuM2fkvY7oqKAlJhERGIugVV52Rx3n+PuX4avlwHfA22AY4Enw82eBI4LXx8LPOWBz4FGZtYK6AMMd/dF7r4YGA4cEa5r6O6fu7sDTyUdq1zqYxIR2bY1NbMxSe8fcveHytvQzDoAuwOjgBbuPidcNRdoEb5uA8xI2m1mWFZR+cxyyjdJiWkLeUkJZbPmbH7DdFBxM3F6cY86AtlWebUHPxS6e8/NbWRm9YGXgUvcfWlyN5C7u5nV2g+5mvJERGJs7ai8GuxjwsyyCZLSM+7+Slg8L2yGI/x3flg+C2iXtHvbsKyi8rbllG+SEpOISMzV8Kg8Ax4Fvnf3O5NWvQ6sHVl3BvBaUvnp4ei8vYCisMlvGHC4mTUOBz0cDgwL1y01s73Czzo96VjlUlOeiEh62xf4A/CtmY0Ly/4G3Aq8YGZnA9OAk8J1Q4GjgElAMXAWgLsvMrMbgNHhdte7+6Lw9fnAE0Au8Fa4bJISk4hIjNX0E2zd/WPY5PC9Q8rZ3oELNnGsx4DHyikfA+xa2ZiUmEREYs7TbOYHJSYRkZjT7OIiIhIbXv3h4ilHo/JERCRWVGMSEYk59TGJiEiM6NHqIiISM6oxiYhIbOhBgSIiIhFTjUlEJM48/SavV2ISEYk53WArIiKx4aTf4Af1MYmISKyoxiQiEmu6j0lERGJGgx9ERCRW0q2PSYlJRCTG3NMvMWnwg4iIxIpqTCIiMafBDyIiEisa/CAiIrGSbn1MSkwiIjHmWNolJg1+EBGRWFGNSUQk5tKsi0mJKc4yMpxBQ75n4bwcBpy1Ay3arebqeybTsHEZP31bj9sv6UBpSQa79l7GnwfMoONOK7mlfyc+Hto46tC3mksHTmfPQ5eypDCLfod0XVd+zFkLOObMQhJlxqgRDXn0ptYcdPwiTjxv/rptOu60iguO2JHJE+pFEXqNyq6TYOArk8jOcTKznI/ebMTTd7Rkt32X8adr55Cd7fz0TS53XtaORNm23wyU17CMv9wxgw5dV+EOd17ajj0OXMaRpy6kaFHwZ+7xW1ox+r2GEUdaBWl4H1OkicnMrgOWAw2Bke7+bpTxxM1xf5zPjEl1qdcgAcDZV8/i1Uda8OEbBVx48zT6nLyQN//bjAWzcxh4WQd+229exBFvfe+8UMDrjzflirunryvbbZ9l7NOniPMO60LJmgzym5QA8P6rBbz/agEAHbquZMCjU7bJpARQstq48sTtWVWcSWaWc+f/JjH2gwZccfcM/nrS9syaXIfTr5jLYSctYthzTaIOt8add/0sxnzQgBvP7UBWdoI6uc4eBy7j1Yeb8dIDzaMOr/rSrMoUiz4md7+2ppKSBWJxnluiacs19DqkiLefbxqWOLvts5SPwtrQuy81YZ8+SwCYN7MOU36ohye2vauq8aPqs2xJ5gZlfU9fyOB7W1CyJvhvLVqY/Yv9DjpuMR++vu3UHH/JWFUcfC9Z2U5mtlNWBiVrjFmT6wDw5Yf12e+ooiiDrBX1GpTxq71W8PazwUVJaUkGK5ZmbmYvibNa/4NtZteY2Y9m9jHQJSx7wsxOCF/fambfmdk3ZnZHWHa0mY0ys6/M7F0zaxGWNzOz4WY2wcweMbNpZtbUzDqY2UQzewoYD7QzsyvMbHR43H8mxfN7M/vCzMaZ2YNmFouf6H7XzeDRm9usSzYNG5exYmnWumaZBXNyaNJyTZQhRqZNp1Xs2ns5d7/xI7e/9BM77lb8i20OOHoJ7/+vUQTR1Z6MDOe+4RMZ/M0EvhpZn4lf1SMzy+ncLfg+9utbRLPWJRFHWfNatl9D0cJMLrtrBve+M5FL7phBndwyAI4+q5D7353IpXdOp35+acSRVp27VXlJRbWamMxsD+AUoDtwFNBro/VNgOOBXdy9G3BjuOpjYC933x14HrgyLB8AvOfuuwAvAe2TDtcZuC9c1yV83zv87D3M7AAz2wk4GdjX3bsDZcBp5cR9rpmNMbMxJb66ul/DZvU+ZAlLCrOZ9G1ejX9WKsrMhAaNyrj46M48cmNrrnlgKsltHV12X8HqlRlMm5gbWYy1IZEwzj+sC6ftsTNduhezXZdV3HLedvz5n7MZ9OaPrFyeQSIRdZQ1LzPT2eFXKxnyVBMuOLwLq4ozOLn/fIY82YSz9t6J8w/bkUXzsjl3wOyoQ60y96ovqai2+5j2B15192IAM3t9o/VFwCrgUTMbAgwJy9sCg82sFZADTAnL9yNIZLj722a2OOlY09z98/D14eHyVfi+PkGi6gbsAYw2M4BcYD4bcfeHgIcAGmYU1Ph/9S49V7DXYUvofVAR2XUS1GtQxp+vm0Few1IyMp1EmdGs1RoWzs2p6VBiqXBONp+8lQ8YE8flkUhAfkHZuk7uA49dwgevbcvNeBtasTSTrz+tT6+DlvHSA8257PgdAOjx62W07VTzF1JRK5yTzYI52Uz8KriQ+3hIPif1n8+SwvVNvG8904Trn5qyqUPEmp5gGzF3LyWo1bwE9AXeDlf9B7jH3X8F9APqVuJwK5JeG3CLu3cPlx3c/dGw/Mmk8i7uft3WOp+qevy2Nvxhz26cse+vuLV/J77+tCH/urgj33zWgP2PCnLvoScs5LN38iOONBqfDstnt32WA0GzXnaOU7QoaIE1cw7ou4QPXtu2m/HyC0rJaxg0V+XUTdDjgOXMmFR33UCQ7JwEJ50/nyFPb/sDHxYvyKZwdg5tt18FQPf9lzP9p7oUNF/fjLnPkUVMnViZPxsx5IBb1ZcUVNs1ppHAE2Z2S/jZRwMPrl1pZvWBeu4+1Mw+ASaHq/KBWeHrM5KO9wlwEnCbmR0ObOoyeRhwg5k94+7LzawNUAKMAF4zs7vcfb6ZFQAN3H3aVjnbrezRW9py9T2TOeOK2fw8IZdhg4OBETt2W8E/Hv6ZBvll7HnoEv5w6Wz6HbpLxNFuHVfdO5Vuey8nv6CU/46ZwNN3tGTY8wVcOnAGD474gZIS4/ZL2hNcY8Cv9lrOgjnZzJ1eJ9rAa1hBixIuv3s6GRmQkQEj38hn1LsNOecfs9nz0KVYBrz5ZBO+/qRB1KHWinv/3oa/3jOdrGxn7vQcBv6lHefdMJvtd1mJO8ybmcOgK9tGHaZUknktN0Ka2TUEyWU+MB34EtiVoNnuE+A1ghqRAXe4+5NmdixwF7AYeA/o5e4Hmllz4DmgBfAZQS2rA9AKGOLuuyZ97sXAOeHb5cDv3f1nMzsZuJqg9lgCXJDUBPgLDTMKfK+sPlvjq0h5XlYWdQjxkaqN+VKj3vWXxrp7z+oco06nNt7mpguqvP+UU6+pdgy1rdbvY3L3m4CbKtikdzn7vEaQsDZWBPRx91Iz25sgYa0GphIku+Rj3A3cXc6xBwODK30CIiK1Lc2ue1J95of2wAvhfUprgD9FHI+IyFaWusO+qyqlE5O7/wTsHnUcIiI1Ks1qTLEalSciIpLSNSYRkW2eJnEVEZHYSbOmPCUmEZHYU40JADP7DxXkaXe/qEYiEhGRtFZRjWlMrUUhIiKbpqa8gLs/mfzezOqtnXxVRERqUZolps0OFzezvc3sO+CH8P1uZnZfjUcmIiJpOYlrZe5j+jfQB1gI4O5fAwfUZFAiIrJeuj2PqVI32Lr7jI2KNHuniIjUiMoMF59hZvsAbmbZwMXA9zUbloiIrJOiNZ+qqkxi+jPBrNxtgNkEzzaq+hzsIiKyZVK0r6iqNtuU5+6F7n6au7dw92bu/nt3X1gbwYmICJhXfdnssc0eM7P5ZjY+qew6M5tlZuPC5aikdVeb2SQzm2hmfZLKjwjLJpnZVUnlHc1sVFg+2MxyNhdTZUbldTKzN8xsQRj8a2bWafOnKyIiKeAJ4Ihyyu9y9+7hMhTAzHYGTgF2Cfe5z8wyzSwTuBc4EtgZ+F24LcBt4bF2IHjY69mbC6gygx+eBV4geCpsa+BFgqfGiohITfNqLps7vPtIYFElozkWeN7dV7v7FGASwcNdewOT3H2yu68BngeONTMDDgZeCvd/Ejhucx9SmcRUz92fdvfScPkvwaPPRUSkxlXjHqbq9U31N7Nvwqa+xmFZGyB5lPbMsGxT5U2AJe5eulF5hTaZmMyswMwKgLfM7Coz62Bm25nZlcDQyp6ZiIhUU/VqTE3NbEzScm4lPvF+YHugOzAHGLg1T2dzKhqVN5bgtNam3H5J6xy4uqaCEhGRJNUbLl7o7j236OPc5619bWYPA0PCt7OAdkmbtg3L2ET5QqCRmWWFtabk7TepornyOlbmBEREZNtiZq3cfU749nhg7Yi914FnzexOgjEHnYEvCCownc2sI0HiOQU41d3dzN4HTiDodzoDeG1zn1+p5zGZ2a4EIy3W9S25+1OV2VdERKqpBm+wNbPngAMJmvxmAgOAA82se/jJUwlbzNx9gpm9AHwHlAIXuHtZeJz+BPe5ZgKPufuE8CP+CjxvZjcCXwGPbi6mzSYmMxsQBr0zQd/SkcDHgBKTiEhNWzuJa00d3v135RRvMnm4+03ATeWUD6Wc8QfuPplg1F6lVWZU3gnAIcBcdz8L2A3I35IPERGRqqvJG2zjqDJNeSvdPWFmpWbWEJjPhp1cIiJSk1I0wVRVZRLTGDNrBDxMMFJvOfBZjUYlIiJpa7OJyd3PD18+YGZvAw3d/ZuaDUtERNLVJhOTmfWoaJ27f1kzIYmISLJU7SuqqopqTBXd6esE8x+lHwcvLd38dpJWVvXdokFH27Tct3XNuk7JVjpOmj32oqIbbA+qzUBERESgkjfYiohIRCo5S/i2RIlJRCTulJhERCRO0m3wQ2WeYGtm9nszuzZ8397M1NMrIlJbavBBgXFUmSmJ7gP2BtbOp7SM4BG6IiIiW11lmvL2dPceZvYVgLsvNrOcGo5LRETWStGaT1VVJjGVmFkm4VdjZs2ARI1GJSIiQGpPxlpVlUlMg4BXgeZmdhPBbON/r9GoRERkPd1guyF3f8bMxhI8+sKA49z9+xqPTEREAqoxbcjM2gPFwBvJZe4+vSYDExGR9FSZprw3CfK1ETxavSMwEdilBuMSEZGQ+pg24u6/Sn4fzjp+/iY2FxGRrU2JqWLu/qWZ7VkTwYiIyEY0Ku+XzOzSpLcZQA9gdo1FJCIiaa0yNaYGSa9LCfqcXq6ZcERE5BdUY1ovvLG2gbtfXkvxiIjIxpSYAmaW5e6lZrZvbQYkIiIbUh/Tel8Q9CeNM7PXgReBFWtXuvsrNRybiIikocr0MdUFFgIHs/5+JgeUmEREZKurKDE1D0fkjWd9QlorzSqWIiIRSrO/uBUlpkygPhsmpLXS7GsSEYmI7mPawBx3v77WIhERkfIpMa2TXvOsi4jEVZolpooerX5IrUUhIiIS2mSNyd0X1WYgIiLyS4b6mEREJG6UmEREJDY0Kk/iJrtOgoGvTCI7x8nMcj56sxFP39GSY84q5PhzFtC64xpO3HUXli5Kj//KS++czp6HLmNJYRb9Du4CQINGpfztgWm0aLuGeTNzuKnfdiwv2na+j/q5q7ni9I/o2GYRuHHbkwdwwiHjaddySbh+DctX5nDODb+la4f5XP6Hj4CgCeiJN3rw0biOADx/83OsXJ1NWcIoK8ug383HR3VK1ZZdJ8EdL05c/3sxtDH/vbM1lw2cyq/2XMaKZZkADLysA5O/q0fb7Vdx2R1T2X7XYp68vTUvP9Qy4jOQisTyt9fMrgOWAw2Bke7+brQRRadktXHliduzqjiTzCznzv9NYvR7DZgwuh6jhm/Pv16eFHWIteqdwQW8/nhTrrh7xrqyk/rP56uP6/PCPS04qf88Tu4/n0dvah1hlFvXhSd/xhcT2jLgwUPJyiyjbk4p/3x4/dik80/4nBUrcwCYMruAfjcdT1kig4L8Yh77x8t8+s12lCWCcU6XDOxL0fK6kZzH1lSy2vjrKTuu+70Y+PIPjHm/IQCP3NyWj4c23mD7ZUsyuX9AO/busySKcKsvzWpMFY3Ki5y7X5vOSSlgrCoOrv6ysp3MbMcdfh5fj3kzcyKOrfaNH1WfZYs3vJ7au89S3n2hAIB3Xyhg7yOWRhFajcjLXcNuO87hzY+D2mFpWSbLV9ZJ2sI5qOdk3h29PQCr12StS0I5WaX4NnvXR9LvRZaTleW4b/pcixZm8+M3eZSVpuj34dVYUlBsakxmdg1wBjAfmAGMNbMngCHu/pKZ3RJaZowAACAASURBVAocQ/BMqHfc/XIzawE8AHQKD3MewUMMh7j7ruFxLwfqu/t1ZnYR8OfwGN+5+ylm9mvg7nB/Bw5w92W1cMqVlpHh3DPsR1p3WMMbTzRh4ld5UYcUK42blrBofjYAi+Zn0bhpScQRbT2tmixjybJcrjrzQ3Zou4iJ05ryn8F7s2pNcL7dOs9l0dJcZs3PX7fPTh3n89czPqRFwXJufuzAdYkK4I5LhuJuvDGyK298tFOtn8/WlJHh/OfN72ndYTVvPNWMiePy6PuHBZx5xSxOu3gOX33SgMdvbUPJmlhff1eK+pgiYGZ7AKcA3Qli+hIYm7S+CXA80NXd3cwahasGAR+6+/Hhs6PqAxvW4Td0FdDR3VcnHeNy4AJ3/8TM6gOryonvXOBcgLrUq8aZVk0iYZx/WBfyGpYx4NEpbNdlJdMm5tZ6HKnBKrxyTjWZmQk6ty/k7uf34fspzbnw5E859Yiveez1ngAc2utnRoS1pbW+n9KcM687ke1aLubqsz5k1Ph2rCnNov+/jqFwSR6NGqxk4CVDmTa3Ed/81CqK09oqEgnjgiN3Jq9hKdc+9DPb7biSx29rw6L5WWTnOBfdOo0Tz5vLs3dvA826aZaY4nIpsT/wqrsXu/tS4PWN1hcRJIxHzew3QHFYfjBwP4C7l7l70WY+5xvgGTP7PUGtCeAT4M6wNtXI3Us33sndH3L3nu7eM5s6G6+uNSuWZvL1p/XpdVCsKnSRW1yYTUHzoJZU0LyEJQtjcb21VSxYnMeCxXl8P6U5AB+O7ciO2xUCkJmRYP8eU3l/dKdy9502tzErV2fRsc1iAAqXBDXtJcty+WhcB3bqsKAWzqDmrViaxdefNaDngUVhzdkoWZPB8Bea0qV78Wb3l/iJS2KqUJgsegMvAX2BtyvYvJQNzyu5p/f/gHsJnjM1OnwY4q3AOUAu8ImZdd2asVdXfkEpeQ3LAMipm6DHAcuZMSn1O6+3ps/facihJwX3gx960iI+G9Yw4oi2nkVL67FgcR7tWgSd9j12ms3U2UGjwB47zWL63HwWLKm/bvuWTZaSmZEAoEXBMtq3LGLuwgbUzSkht84aAOrmlNBr55lMmV1R40K85ReUkNcwuIbMqZOgx/7LmPFz3XUXKODs3WcJUyduA78r1elfStGaVlwuLUcCT5jZLQQxHQ08uHZl2MRWz92HmtknwORw1QiCfqV/JzXlzSN4ZEcTgpF9fYG3zSwDaOfu75vZxwRNh/XNrIm7fwt8a2a9gK7AD7VwzpVS0KKEy++eTkYGZGTAyDfyGfVuQ449ewEnnreAguYlPPDuRL54ryH/vrxd1OHWuKvum0a3vZeTX1DKf8d8x9MDWzD4nuZc88A0jjhlEfNnBcPFtyV3P7cvfz/7fbKzEswubMCtT/wagIN7/cyILzZsxuvWeR6nHjGM0rIM3I27nt2XouV1adV0KTeeNxwImgff/WIHvpiQuj8vBc1LuOzOqWRmgmU4I4c05osRjbj1uR/Jb1KCGUyeUI9Bf2sPQONmJQwa8j316pfhCeO4s+fT75BdKF6eGfGZVE669TGZezzOeKPBD9MJ+pl2BYYQNLe9RlD7MeAOd38yHPzwEMHghzLgPHf/LGyWuxiYRZDEpgI3Ae8D+eEx/uvut5rZf4CDgAQwATjT3VdvKs6GVuB7mqYRlA2t6ts76hBiI/ftL6MOITaGlzw/1t17VucYuS3b+fZ/uLTK+0+449Jqx1Db4lJjwt1vIkgem/KL33x3nwccW075IIKBERvbr5xtL9yCMEVEal261ZhSoo9JRETSR2xqTCIisglpVmNSYhIRibMUHl1XVUpMIiIxZqTf48TVxyQiIrGixCQiEnc1eIOtmT1mZvPNbHxSWYGZDTezn8J/G4flZmaDzGySmX1jZj2S9jkj3P4nMzsjqXwPM/s23GeQmW22AqjEJCISc+ZVXyrhCeCIjcquAka4e2eCiQyuCsuPBDqHy7mEU8KZWQEwANiT4NaeAWuTWbjNn5L22/izfkGJSUQk7mqwxuTuI4FFGxUfCzwZvn4SOC6p/CkPfA40MrNWQB9guLsvcvfFwHDgiHBdQ3f/3IPZHJ5KOtYmafCDiEjcVW9UXlMzG5P0/iF3f2gz+7Rw9znh67lAi/B1G4LHEq01MyyrqHxmOeUVUmISEdm2FVZnSqLwUUO1OmBdTXkiInFWjf6laqSTeWEzHOG/88PyWUDy7L9tw7KKytuWU14hJSYRkbir/cdevE4wqTbhv68llZ8ejs7bCygKm/yGAYebWeNw0MPhwLBw3VIz2yscjXd60rE2SU15IiIxV5MNaWb2HHAgQV/UTILRdbcCL5jZ2cA04KRw86HAUcAkgge2ngXg7ovM7AZgdLjd9e6+dkDF+QQj/3KBt8KlQkpMIiJxV4OJyd1/t4lVv3i+Tziy7oJNHOcx4LFyyscQPMKo0tSUJyIisaIak4hIzKXb85iUmERE4kyzi4uISOykWWJSH5OIiMSKakwiIjFmqI9JRETiRolJRETixDy9MpMSk4hInKXhqDwNfhARkVhRjUlEJOY0+EFEROJFiUlEROIk3WpM6mMSEZFYUY2pKjIyo44gHhJlUUcQG7nDv446hNh4e/qYqEOIjcxWW+lAaVZjUmISEYmz6j0iPSUpMYmIxJ0Sk4iIxEU6zpWnwQ8iIhIrqjGJiMSd5soTEZE4SbemPCUmEZE4S8NJXJWYRERizhJRR1C7NPhBRERiRTUmEZG4U1OeiIjEiQY/iIhIfDhpN1xcfUwiIhIrqjGJiMScmvJERCRelJhERCQu0nESVyUmEZE4c9fgBxERkSipxiQiEnNqyhMRkXhRYhIRkThRjUlEROLDgUR6ZSYNfhARkVhRjUlEJO7Sq8KkxCQiEnfqYxIRkXjRDbYiIiLRUY1JRCTm1JQnkcuuk2Dgyz+SneNkZjofDW3E0wNb89f/TKFzt2LKSoyJ4/K4+6r2lJUa4Jx3/Ux6H7yUVSuNgX/pwKTx9aI+jRr35KjvWLk8k0QCykqNC4/cMeqQalxeg1IuuW0KHXZciTvcdWVHvv+qAQC/OWcO514zg5N67M7SxdkAdNtzKf2unU5WllO0OIsrT9kpyvC32PxZ2dx+cXuWLMgGc476/UKOP6eQn8fnMuiqtqxZlUFmltP/lpl03b2Y915pzAv3NscdcvMSXHjrDLbfZRUAy4syuevydkz9oS5mcOmd09m5ZzFP39GSt54tIL+gDICzrp5N70OWRXnaG3I0+CHuzOxA4HJ371vBNt2B1u4+dAuP/UF47DHVCrKaSlYbV57UmVXFmWRmOXe+OpHR7+fz3qsF3HZhBwCuumcqR/6ukCFPN6PXwUtp03E1Z+23M117FHPhLdO5+OiuUZ5CrbnyxO1Zuijlfoyr7M8DpjH2w3xuOr8zWdkJ6tRNANC01Wr22L+IebNy1m2b16CUC26Yyt/P7MKC2XXIb1ISVdhVlpnlnHvtbDp3W0nx8gz6H7EjPQ5YxiM3tuL3l86l18HL+GJEAx69sTW3vzyJFu1Wc/vLk2jQqIzR7zXg7ivbMejNnwC4/9o29DxwKf94eCola4zVK9f3ZBz/pwWceN6CqE6zQsHs4umVmWLZx2Rm1f1L0x04amvEEg1jVXEmAFlZTmaW4w6j38sn/DFl4rh6NG21BoC9Dy/i3ZcKAOOHL/PIa1hGQfPU+yMkFavXoJRf9V7G24ObAVBaksGKZcGvSr9/TOeRW9tvcGV90LEL+XRYAQtm1wGgaGF2rcdcXU1alNK520oA6tVP0G6H1RTOycYMViwLfkdWLM2koEXw875Lr2IaNApqPl17FFM4JzvcJoNvP8/jiFMXAZCd49TPL6vt06m6RDWWFFRjl5pm1gEY4u67hu8vB+oDBwKjgIOARsDZ7v6RmZ0J/CbcJtPMjgL+A+wKZAPXuftrG31Gb+BuoC6wEjgLmAJcD+Sa2X7ALcCQ8o5lZrnA48BuwA9Abg18FVWSkeHc89YPtO6wmjeebMbEr/LWrcvMcg757SIeGNAOgKYt17Bg9vor5cI5OTRpuYZF81PvD9EWcePm5yaDw5tPN+GtZ5pEHVGNatl2NUWLsrns9il03KmYSePzuP+f7dl9v6UsnJvDlO83bL5t03EVWdnOv577nty8Mv73REtGvNI0ouirb+6MHH4en0vXHsX8+fpZ/O132/Pw9a2DJs3Xf/rF9m8/V0Cvg4ImubnT65DfpJSBf2nP5Al16dxtJefdMIu69YK/3G883owRLxXQuVsx5w6YvS65STSiqjFluXtv4BJgQFJ5D+AEd/81cA3wXrjdQcDtZpa30XF+APZ3992Ba4Gb3X1N+Hqwu3d398EVHOs8oNjddwrj2KO8YM3sXDMbY2ZjSli9db6BzUgkjPP77MRpvXalS/cVbNdl5bp1F948nfGj6jP+i/q1EktcXXrcDvTvsyPXnNaRY84sZNc9l0cdUo3KzHJ22GUFQ55pTv++u7KqOIM/XDKLU86fzVN3tSl/+11X8I8/7sg1Z3Th1P6zaNNxZTlHjr+VKzK44ZwO/Pn6WeQ1SDDkyab0++csnhn7Hf2um82dl7bfYPtxn9Rn2HNNOPua2QCUlcGkb+vR9/RC7hv+I3XrJRh8T3MA+p5RyOOffcd9wydS0KKEh/7ZutbPb3PMvcpLKooqMb0S/jsW6JBUPtzdF4WvDweuMrNxwAcEtaINf/ogH3jRzMYDdwG7bOLzNnWsA4D/Arj7N8A35e3s7g+5e09375lNnUqe4taxYmkWX3/agF4HLgXgtL/MIb+glAf/2XbdNoVzc2jWes26901brWHh3JxfHGtbs3BuUCMsWpjNJ2/n03X34ogjqlmFc3IonJvDxHHBBclHbxWw/S7FtGy7mvuHjufJj8bRtOUa7nljAo2brqFwTg5jR+azemUmSxdnM/6LBnTaKfUSU2kJ3HBOBw7+zWL2O6oIgOEvFqx7fcDRS/hx3Pra4uTv6vLvy9tx3eNTaBgOaGjaqoRmrUro2iP4Gdmv7xImfRs0kDRuVkpmJmRkwJGnLWLiuJgNHPJqLimoJhNT6UbHr5v0em21o4wNmxNXJL024Ldhrae7u7d39+83+owbgPfD5sKjN/qMZJU5VmzkF5SQ17AUgJy6CXrsv5QZk+pyxO8K6fnrpdzSvyPutm77z9/J59ATFgFO1x4rKF6Wuc0349XJLSM3r2zd6z1+vYypP2zqv3/bsLgwhwVzcmjbKUguu+9TxM8T6nFKrx6csX93zti/O4Vzc+h/9C4sLszhs+GN2aXncjIynTp1y+jSfQXTJ6XWd+QOd17WnnadV/PbfusHJzRpUcI3nwUJetzH9WndMfiTMn9mNtef05ErBk2j7fbrWzcKmpfStPUaZkwKLizHfdSA9p2D9Qvnrf8T9Olb+XTosqrGz2vL+Pqn2FZlqQQzm2pm35rZODMbE5YVmNlwM/sp/LdxWG5mNsjMJpnZN2bWI+k4Z4Tb/2RmZ1T1jGtyONM8oLmZNQGWA32Bt7dg/2HAhWZ2obu7me3u7l9ttE0+MCt8fWZS+TKgQSWONRI4FXjPzHYFum1BfDWmoEUJl981jYxMJ8Ng5JDGjBqRz9CpXzJvZg7/fm0iAJ+81Yhn/t2KL95rSK+Di3j84wmsXpXBwEu3i/gMal7jZqUMeHQqEDRZvf9qY8Z80DDaoGrBfQO248q7fiY7x5kzvQ53XtFpk9vO+DmXsSPzuf+tb/GE8fbgZkz7MWa1gc2Y8EUeI14qoONOKznv0C5AMJz7kttncP+1bSgrM3LqJLjk9hkAPHNXS5YtzuSeq4P+18ws5563fwTgghtncVv/7SgtMVq2X8Nld00H4NEbW/PzhFzMoEXbNVz0rxkRnGnFauk+poPcvTDp/VXACHe/1cyuCt//FTgS6BwuewL3A3uaWQFBl0hPgrraWDN73d0Xb2kg5jXYBmlmFwEXEySPycBUgsEPl7v7GDNrCoxx9w7h4Iee7t4/3DcX+DewD0HNa4q7900eLm5mewNPEtS03gR+Hx6rgCAZZRMMfnh9E8dKHvzwPdAGuKCi4eINrcD3zDx8a31FqS2hDuK1rE7tNvHG2dtTRkUdQmxktpo01t17VucYDRu08d49Lqjy/iNGXrPZGMxsKsHf38KksonAge4+x8xaAR+4exczezB8/VzydmsXd+8Xlm+w3Zao0RtA3H0QMKiC9YWEfUzu/gTwRNK6lUC/cvb5gKCfCHf/DEi+q/LvYfkioNdGu5Z3rJXAKZs9ERGRKFWvAtF0bfNc6CF3f2jjTwDeMTMHHgzXt3D3OeH6uUCL8HUbILlaOTMs21T5FkufOxNFRFKRg1XvfqTCStTa9nP3WWbWHBhuZj9sEELQBVJrQylieYOtiIgkqeHBD+4+K/x3PvAq0BuYFzbhEf47P9x8FtAuafe2YdmmyreYEpOISBozszwza7D2NcHtNeMJ+ubXjqw7A1g7wcHrwOnh6Ly9gKKwyW8YcLiZNQ5H8B0elm0xNeWJiMRdzTaitQBeNTMIcsKz7v62mY0GXjCzs4FpwEnh9kMJpnybBBQTzLiDuy8ysxuA0eF21yfdl7pFlJhERGKuJmdwcPfJBCOTNy5fCBxSTrkD5Q4TdPfHgMeqG5MSk4hI3KXo1EJVpcQkIhJnTsrOEl5VGvwgIiKxohqTiEiMGak7S3hVKTGJiMSdEpOIiMSKEpOIiMSGBj+IiIhESzUmEZGY0+AHERGJFyUmERGJj8rPEr6tUB+TiIjEimpMIiJx5qRdjUmJSUQk7tJsuLgSk4hIzGlUnoiIxEuaJSYNfhARkVhRjUlEJM4cSKRXjUmJSUQk1tLvPiYlJhGRuFNiEhGRWEmzxKTBDyIiEiuqMYmIxJkGP8jmLGNx4btlg6dFHQfQFCiMOoiYiP67WBXppyeL/LvIbBXlp28g8u8C2K76h3Dw9Jr6QYlpC7l7s6hjADCzMe7eM+o44kDfxXr6Ltbbpr4L9TGJiIhERzUmEZE4Ux+TpJCHog4gRvRdrKfvYr1t57tIs6Y8JaYU5e7bzi9dNem7WE/fxXrb1HehxCQiIvGRflMSafCDiIjEimpMIiJx5kBC9zFJjJlZPXcvjjqOKJmZAacBndz9ejNrD7R09y8iDq3WmVkesNLdE2a2I9AVeMvdSyIOrVaZWY+K1rv7l7UVS41Is6Y8JaYUYWb7AI8A9YH2ZrYb0M/dz482skjcBySAg4HrgWXAy0CvKIOKyEhgfzNrDLwDjAZOJkjc6WRg+G9doCfwNWBAN2AMsHdEcW0daZaY1MeUOu4C+gALAdz9a+CASCOKzp7ufgHhREDuvhjIiTakyFhYg/4NcJ+7nwjsEnFMtc7dD3L3g4A5QA937+nuewC7A7Oija66PLiPqapLClJiSiHuPmOjorJIAoleiZllErS+Y2bNCGpQ6cjMbG+CGtKbYVlmhPFErYu7f7v2jbuPB3aKMB6pAjXlpY4ZYXOem1k2cDHwfcQxRWUQ8CrQwsxuAk4A/h5tSJG5BLgaeNXdJ5hZJ+D9iGOK0jdm9gjw3/D9acA3EcZTfQ6eZpO4mqdZ22WqMrOmwN3AoQRt5+8AF7v7wkgDi4iZdQUOIfguRrh7uibpdcwsA6jv7kujjiUqZlYXOI/1zdwjgfvdPT7zv2+h/KxmvnfD46q8/7DFj4xNtclsVWNKEe5eSPp1aFekKVDs7o+bWTMz6+juU6IOqraZ2bPAnwmadUcDDc3sbne/PdrIouHuq8zsAWCou0+MOp6tJs0qEOpjShFmtqOZjTCz8eH7bmaWls1XZjYA+CtBExZANuubbtLNzmEN6TjgLaAj8IdoQ4qOmR0DjAPeDt93N7PXo41KtpQSU+p4mOAPcQmAu38DnBJpRNE5HjgGWAHg7rOBBpFGFJ3ssM/xOOD18P6l9Lq83tAAoDewBMDdxxEk69TlHtxgW9UlBSkxpY565dxAWhpJJNFb40Hn6NpReXkRxxOlB4GpQB4w0sy2A9K2jwkocfeijcpSP1G7V31JQepjSh2FZrY96/8Yn0Bwz0Y6esHMHgQamdmfgD8S1CjTjrsPIhiluNY0MzsoqnhiYIKZnQpkmlln4CLg04hjqjZP0ZpPVSkxpY4LCJ4v09XMZgFTSMPBEOF0RIMJpt5ZCnQBrnX34ZEGFhEzu3YTq66v1UDi40LgGmA18CwwDLgh0oiqLXVrPlWlxJQCwptJz3f3Q8Nmqwx3XxZ1XFFwdzezoe7+KyAtk9FGViS9rgv0JX3vbwP4P3e/hiA5AWBmJwIvRheSbCklphTg7mVmtl/4esXmtk8DX5pZL3cfHXUgUXP3gcnvzewOglpCurqaXyah8spShx6tLjH2VTjs9UWSrpLd/ZXoQorMnsBpZjaN4LswgspUt2jDioV6QNuog6htZnYkcBTQxsyS+9wasi0MEkqzmR+UmFJHXYIJXA9OKnMgHRNTn6gDiAsz+5b1o84ygWakZ//SbIJZxI8BxiaVLwP+EklEW4kDrhqTxJG7nxV1DDFSXv9aWva5EfQprVUKzHP31K8hbKFwtv2vzezZdHsW1bZIiSlFbNQ8sVYRMMbdX6vteCL2JdAOWEzQjNcImGtm84A/ufvYinbexrQCJqwdDGNmDcxsZ3cfFXFcUelgZrcAOxO0MgDg7p2iC6ma3NOuKU832KaOukB34Kdw6UbQl3C2mf07ysAiMBw4yt2bunsT4EhgCHA+wUME08n9wPKk9yvCsnT1OMH5lwIHAU+xDUxX5Qmv8pKKVGNKHd2Afd29DMDM7gc+AvYDvq1ox23QXu7+p7Vv3P0dM7vD3fuZWZ0oA4uAedIjAsJHrKfz73Wuu48wM3P3acB1ZjYW2NT9XqkhzWpM6fwDnGoaEzxWfe10K3lAQTiUfHV0YUVijpn9FXg+fH8yMC+83yu9foNhspldxPpa0vnA5Ajjidrq8PEfP5lZf4Kn19aPOKZqWcbiYe/6S02rcYjCrRZMLdHzmFKEmZ1N8DC8Dwj6VQ4AbgaeA65z9yuii652hc+mGkBQWwT4BPgnQdJu7+6TooqttplZc4IpiQ4mGMA1ArjE3edHGlhEzKwXwQ3GjQhmfMgH/uXun0camGwRJaYUYmatCR5p8D3BVeBMdx8ZbVQiIluXmvJShJmdQ/A49bYEz5vZC/iMDe9rSgtm1gy4EtiFDUdepeN38S/gRmAlwTOIugF/cfeU7/DfEmb2BhXMIu7ux9RiOFJNSkyp42KgF/C5ux8UPlr85ohjisozBBO59iV4eusZwIJII4rO4e5+pZkdT/D4i98QPE48rRITcEfUAcjWo8SUOlaFj43GzOq4+w9m1iXqoCLSxN0fNbOL3f1D4EMzS9d589b+Dv8f8KK7FwUTsKeX8OcAADPLJehr3HYerZ5mdB9T6phpZo2A/wHDzew1YFrEMUVl7Z39c8zs/8xsd6AgyoAiNMTMfgD2AEaEzZyrIo4pMmZ2NHq0esrT4IcUZGa/Jhht9La7r4k6ntpmZn0J7uFqB/yHYKLOf7p7Wv4BMrMCoCi8daAe0NDd50YdVxTCe5YOBj5w993Dsm/Dx6RIilBTXgpKbrZIR+4+JHxZRHB3f9oxs4Pd/T0z+01SWfIm6Ti5L4SPVt/ou9DVd4pRU56kHDP7l5k1NLNsMxthZgvM7PdRx1XLDgj/PZpgEMjG/6arDR6tbmb/YRt4tHq6UWKSVHS4uy8l+AM8FdgBSJsbjEPLzOxSYHzSMoFgeqrxUQYWsQsJbiNY+2j1IuCSSCOSLaamPElFGom2fpqdLgS3EbxGMCPI0cAXUQUVpXBKqjfd/SCSHq0uqUeJSVLR2pFoK4Hz0nEkmrv/E8DMRgI9kh57cR3wZoShRSYc/JEws3x3L9r8HhJXGpUnKUkj0QJmNhHo5u6rw/d1gG/cPS3vcQtvo9id4NEoK9aWu/tFkQUlW0w1JkkZGolWrqeAL8zs1fD9ccAT0YUTuVdIz5+DbYpqTJIyzOw6d7/OzB4nGAJsyf+6+x8jDTAiZtYD2D98O9Ldv4oyHpHqUmKSlGFml/HLhET4Gne/M6LQJCbMbF/gOmA7ghahtRctqfto9TSkpjxJJRqJJpvzKPAXYCxQFnEsUkWqMUnKCUei/V/SSLQGBMOED6h4T9nWmdkod98z6jikelRjklTUAkieI3BNWCbyvpndTjAAYvXaQnf/MrqQZEspMUkq+v/27iU0riqO4/j31ypaqxarEQSf+KCGolWr1oAhiojVhVSUQl1ppVawhYJrHwVBUOhGxEcQEVGkVEURk2AlNBXFaGlrkyJdRFwIIm19pNZF69/F/Y8zDJNkMhTmzvj7bHLn3HPuOXcW8+c8co5XotlMKr2lm/JvZT7yf3eIZCdzYLKOExHPS/qM6kq0R7wSzdJogzTPV3QYBybrSDk04+EZqzddc30mxX6KB9vUFmuRFz+YWdfKnTCGI2Kg3W2x5nl3cTPrZmcBF7e7ETY/Hsozs64h6Xuqc0oLgR5ga/taZK3wUJ6ZdQ1Jl9V8PAH8EhEn2tUea40Dk5mZlYrnmMzMrFQcmMzMrFQcmKwjSTopaa+kA5K252GBrT7rLUkP5vWgpN5Z8g5I6muhjh8lXdBsel2e6dnuN8j/rKSn5ttGs7JwYLJOdTwiVkTEcoq98jbW3pTU0orTiHgsIiZnyTIAzDswmVnzHJisG4wBV2VvZkzSx8CkpIWSXpQ0Lmm/pMcBVHhZ0g+SPgcurDxI0qiklXl9j6Q9kvZJ2inpcooAuCV7a7dL6pG0I+sYz/OAkHS+pBFJE5IGqZ4dNSNJH0n6LstsqLu3LdN3SurJtCslDWWZMUnLTsWXadZu/j8m62jZM1oNDGXSjcDyiJjKH/ffI+Lm3AHgS0kjwA0UZzr1UuxKPgm8WffcHuANRQrotAAAAfhJREFUoD+ftTQijkh6FZiOiJcy37vAtojYLelSYBi4FngG2B0RWyXdB6xv4nUezToWAeOSdkTEYWAx8G1EbJH0dD77SeB1YGNEHJJ0K/AK3qzUuoADk3WqRZL25vUYxQFxfcA3ETGV6XcD11Xmj4AlwNVAP/BeRJwEfpb0RYPnr6I4pnwKICKOzNCOu4Be6b8O0bmSzs46Hsiyn0o62sQ7bZa0Jq8vybYeBv4B3s/0d4APso4+YHtN3Wc0UYdZ6TkwWac6HhErahPyB/pYbRKwKSKG6/LdewrbsQBYFRF/N2hL0yQNUAS52yLiL0mjFJuQNhJZ72/134FZN/Ack3WzYeAJSacDSLpG0mJgF7A256AuAu5oUPZroF/SFVl2aab/CZxTk28E2FT5IKkSKHYB6zJtNXDeHG1dAhzNoLSMosdWsQCo9PrWUQwR/gFMSXoo65Ck6+eow6wjODBZNxukmD/aI+kA8BrFKMGHwKG89zbwVX3BiPgV2EAxbLaP6lDaJ8CayuIHYDOwMhdXTFJdHfgcRWCboBjS+2mOtg4Bp0k6CLxAERgrjgG35DvcSXXvt4eB9dm+CeD+Jr4Ts9LzlkRmZlYq7jGZmVmpODCZmVmpODCZmVmpODCZmVmpODCZmVmpODCZmVmpODCZmVmpODCZmVmp/AtRWr3ZWnfuMwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x432 with 2 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "###########   VALIDATION\n",
        "\n",
        "# compute score on validation\n",
        "headline_train, body_train, labels_train, headline_val, body_val, labels_val = get_dataset_splitted()\n",
        "\n",
        "BERT = False\n",
        "\n",
        "if BERT:\n",
        "  start_time = time.time()\n",
        "  pred_labels = get_predictions(headline_val, body_val, model_loaded)\n",
        "  inference_time = time.time() - start_time\n",
        "else:\n",
        "  vectorizer = get_vectorizer(SEQ_LENGTH)\n",
        "  vect_head_val = vectorizer(np.array([[s] for s in headline_val])).numpy()\n",
        "  vect_body_val = vectorizer(np.array([[s] for s in body_val])).numpy()\n",
        "\n",
        "  start_time = time.time()\n",
        "  pred_labels = get_predictions(vect_head_val, vect_body_val, model_loaded)\n",
        "  inference_time = time.time()\n",
        "\n",
        "gold_labels = np.array([LABELS[i] for i in labels_val])\n",
        "print(f'Inference time: {inference_time}')\n",
        "\n",
        "\n",
        "fig, ax = plt.subplots(figsize=(6, 6))\n",
        "ConfusionMatrixDisplay.from_predictions(gold_labels, pred_labels, xticks_rotation='vertical', ax=ax)\n",
        "score_test = score_metrics(gold_labels, pred_labels)\n",
        "print(get_confusion_matrix(gold_labels, pred_labels))\n",
        "\n",
        "\n",
        "unrelated_num = sum(map(lambda x : x == 3, labels_val))\n",
        "max_score = len(labels_val) - (unrelated_num * 0.75)\n",
        "print(f'Score test: {score_test}/{max_score}, so {(score_test*100)/max_score}%')\n",
        "print()\n",
        "\n",
        "f1_summary = f1_score(gold_labels, pred_labels, average=None)\n",
        "f1_micro = f1_score(gold_labels, pred_labels, average=\"micro\")\n",
        "f1_macro = f1_score(gold_labels, pred_labels, average=\"macro\")\n",
        "print(f1_summary)\n",
        "print(f1_micro)\n",
        "print(f1_macro)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 525
        },
        "id": "rLdA9UltsRmD",
        "outputId": "886887fc-d92b-4cf0-ca2a-faaaa5ff2d51"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Inference time: 1643631135.8375232\n",
            "[[579, 34, 63, 60], [113, 31, 11, 13], [11, 6, 1626, 139], [116, 2, 253, 6938]]\n",
            "Score test: 4030.0/4513.25, so 89.29263834265772%\n",
            "\n",
            "[0.74469453 0.25726141 0.87068273 0.95967909]\n",
            "0.9178589294647324\n",
            "0.7080794420202039\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAF8CAYAAACJym8NAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd5xU9fX/8dfZArv0jlQBRRARAVHAFluwRk1i+1lCEhMbEWssXxM1qGl2TdQQe4yJLcYaEVtQYwEsNEGQjnTYZSm7bDm/P+5dGNbdZdky987s+/l4zGNnPnPn3jPDsmc+n3vu52PujoiISLJlRB2AiIg0TkpAIiISCSUgERGJhBKQiIhEQglIREQioQQkIiKRyIo6ABERqdoxRzT3tetKa/36qdOKJrj7sfUYUr1RAhIRibE160r5eEL3Wr8+u8vXHeoxnHqlITgREYmEekAiIrHmlHpZ1EE0CCUgEZEYc6CM9JwyTQlIRCTmykjPHpDOAYmISCTUAxIRiTHHKU3TVQuUgEREYk7ngEREJOkcKFUCEhGRKKRrD0hFCCIiEgn1gEREYsxBRQgiIhKN9LwKSAlIRCTWHFcRgoiIRMChND3zj4oQREQkGuoBiYjEWDAZaXpSAhIRiTWjFIs6iAahBCQiEmMOlOkckIiISP1RD0hEJOY0BCciIkkXTEaqBCQiIhEocyUgERFJsnTuAakIQUREIqEekIhIjDlGaZr2FZSARERiTueAREQk6dL5HJAS0C5qkpnruVmtog4jFnxrcdQhiMRaAevXuHvHuu3FKHUNwQmQm9WKg3Y7K+owYqFk6bKoQ4gPS88/EFI3b5Y+vSjqGOJMCUhEJMaC2bDT8wuOEpCISMzpHJCIiCSde/qeA0rPdyUiIrGnHpCISMyVaQhORESSLbgOKD0Hq5SARERiLX3PASkBiYjEWDqXYafnuxIRkdhTAhIRiblSt1rfasLM2pjZc2Y228y+NLORZtbOzCaa2dzwZ9twWzOze81snplNM7OhCfsZHW4/18xG7+y4SkAiIjFWvhxDbW81dA/wurv3B/YDvgSuBd5y977AW+FjgOOAvuHtfOABADNrB9wIDAcOBG4sT1pVUQISEYm5Ms+o9W1nzKw1cBjwMIC7b3X3POBk4PFws8eBU8L7JwNPeOAjoI2ZdQGOASa6+zp3Xw9MBI6t7tgqQhARibEklGH3BlYDj5rZfsBU4FKgs7svD7dZAXQO73cDliS8fmnYVlV7ldQDEhFJbx3MbErC7fwKz2cBQ4EH3H0IsIntw20AuLsT5MJ6pR6QiEiMOTUvJqjCGncfVs3zS4Gl7v5x+Pg5ggS00sy6uPvycIhtVfj8MqBHwuu7h23LgMMrtL9bXWDqAYmIxFwZGbW+7Yy7rwCWmFm/sOkoYBbwElBeyTYaeDG8/xLwo7AabgSQHw7VTQBGmVnbsPhgVNhWJfWARERizJ1kzIRwCfB3M2sCzAd+QtBBecbMzgMWAaeH274GHA/MAzaH2+Lu68zsZmByuN04d19X3UGVgEREGjl3/xyobJjuqEq2dWBMFft5BHikpsdVAhIRiTXTbNgiIpJ8TlKG4CKhBCQiEnNajkFERJLOMcrqVoYdW+mZVkVEJPbUAxIRiTkNwYmISNI51GhS0VSkBCQiEmtGaZqWYadnWhURkdhTDyimHnnhbbZszqKszCgtNS778SFcc8undN99EwDNWxSzaWM2l5x7KFlZZfziuun07Z9PmcP4O/dh+qftI34H9S+7aRl3PD+P7KZlZGbCe6+25m93dNn2/EXjlnLMmes4Za9BEUaZPM1blXD5bYvp1W8L7nDnlbtzwJEbGHlMHl5m5K3J4vYrdmfdyiZRh9rgKvssls7P4f/uX0DnHltZuaQJt17Um435qfcnT0NwEonrLh7Bhvztfzz+8KttK99y3thZbN6UDcAxpywGYMzZh9G6bRHj7v6Ey358CJ5mpZvFRcbVp+9B4eZMMrOcO1+Yy+R3WjH70+b0HbSZFm1Kow4xqS76zVKmvNuKWy7oQ1Z2GU1zy1j0VS5P3N4VgJN/uopzLlvBvdf1jDjShlfZZ3HmJSv47IOWPPPn3Th9zArOGLOSh39b7fI0saUhuBRlZmmYZJ1Dj17Of98I/tD07L2RL6YEPZ789U3ZWJBN373zowywgRiFmzMByMpyMrMdd8jIcH7+6294+JauEceXPM1alrLv8I28/o/g372kOINNG7LYvDFz2zY5uWV4va/gEj9VfRYjR+Xz5rNB25vPtmfkMXlRhllr7tagK6JGKWX+OJvZvwnWoMgB7nH38eEsrdcAecAXQJG7/8LMHgMKgSHAB2b2Z+DPQEeC2Vt/7u6zzawj8CBQ/hXxMnf/IJnvqyoO3HxvsDzHf17Yndf/vf1b7D6D15G3rinfLGkOwIK5rRhx6Er++0ZXOnYqZM/++XTovIWvZrWJIvQGlZHh/On1OXTttZWXH+vAnM+ac8p5q/nwjVasW5UddXhJs1uPIvLXZXHlnYvoM2ALc6c344EbulO0JZMfX72Mo09dx6YNmVx9et+oQ21wVX0WbTuUbPudWLcqi7YdSiKOtPbSdSqeVHpXP3X3/QlmbB1rZt2AXwMjgIOB/hW27w4c5O5XAOOBS8LXXwXcH25zD3CXux8A/BB4qLIDm9n55asJbi3dUt/vq1JXn38Ql44+lBsuO5ATTl3IPoPXbnvuO6O+2db7AXjj5e6sWZXLPY99wPlXzOLL6W0pK03PLntZmXHxqP6cPWwA/YZsZuDwjRx6Yh4vPtIx6tCSKjPL2XPgZl75W0fGHLs3hZszOGPMSgAe+2M3zjlwX95+oR0n/WR1xJE2vOo+i+2sUfQGU00qJaCxZvYF8BFBT+hc4L/uvs7di4FnK2z/rLuXmlkL4CDgWTP7HPgLUH7m+mjgT2H7S0CrcPsduPt4dx/m7sOaZOY2zLurYO3qHCAYUvvw3d3ot08wfJCRWcZBR6xg0pvbT76XlWbw17sHcMm5h3LzL4fRokUxy8LeUbratCGLLz5owX4HbaRrryIe/WAWj380k6a5ZTz6/qyow2twa5Y3YfXyJsz5LPh3fv/Vtuy57+Ydtnn7hXYcclxqDjvtiqo+i/VrsmjXqRiAdp2KyVubMgM+O3CgLJwRuza3OEuJBGRmhxMki5Huvh/wGTB7Jy/bFP7MAPLcfXDCbe+E50YktHdz940N8R52RdOcEnKblWy7P3T4ahZ93RKAIQesYenCFqxdtT0RNm1aStOcYPvBB66mtDSDJQtaJj/wBta6XQnNWwXvs0lOGUMPK2De9Fz+35CBjB6xD6NH7EPRlgx+csiAiCNteOtXZ7Pmm2y69ykEYPAhG1g8N4euvQu3bTPymDyWfJ0TVYhJU9Vn8dHE1hx9WjBycPRpa/nwjdZRhlkHRqln1PoWZ6nylaA1sN7dN5tZf4Jht+bAd8KlXwsIhtCmV3yhu28wswVmdpq7P2tmBgxy9y+ANwhWArwNwMwGhwszRaptu61c/8cpAGRmOv+d0JWpH3UC4LDvLt9h+A2gdbsibr7nE7ws6DndftN+SY85Gdp1LuaquxeTkeFkZMCkl9vw8Zup+kel7v786x5cc99CspqUsWJRU+64cncuv20x3fsUUuawammTRlEBB5V/FmZw/YMLOPbMtaxaGpRhp6KgDDvePZnaMk+BgVEzawr8G+gFzAHaADcBewG/BNYR9IiWuvv1YRHCK+7+XPj63sADBENv2cA/3X2cmXUgKE7YmyAZT3L3C6uLpXXTzn7QbmfV91tMSSVLl0UdQnxYvL9pSjTeLH16qrtXttJojXXZp62P/se3FiatsT/s93ydY2goKdEDcvci4LiK7WY2JayGywJeIEhSuPuPK7x+AXBsJftdA5zREDGLiEj1UiIBVeMmMzuaoDT7DcIEJCKSLtJ5PaCUTkDuflXUMYiINLSy1KgX22UpnYBERNKdO5SmaQ8oPdOqiIjEnnpAIiIxp3NAIiKSdEERQnoOVikBiYjEXLoux6AEJCISY+k8E0J69utERCT21AMSEYk1nQMSEZGIxH1ZhdpSAhIRibF0vhBVCUhEJObSdQguPd+ViIjEnnpAIiIxptmwRUQkMipCEBGRpNOFqCIiIvVMPSARkZhTFZyIiCSfB0UItb3VhJktNLPpZva5mU0J29qZ2UQzmxv+bBu2m5nda2bzzGyamQ1N2M/ocPu5ZjZ6Z8dVAhIRiTEnKEKo7W0XHOHug919WPj4WuAtd+8LvBU+BjgO6BvezgcegCBhATcCw4EDgRvLk1ZVlIBERGKuoXtAVTgZeDy8/zhwSkL7Ex74CGhjZl2AY4CJ7r7O3dcDE4FjqzuAEpCIiDjwhplNNbPzw7bO7r48vL8C6Bze7wYsSXjt0rCtqvYqqQhBRCTG6qEMu0P5eZ3QeHcfX2GbQ9x9mZl1Aiaa2ewdYnB3M/O6BFEZJSARkZirYwJak3Bep1Luviz8ucrMXiA4h7PSzLq4+/JwiG1VuPkyoEfCy7uHbcuAwyu0v1vdcTUEJyISY+VT8TTUOSAza25mLcvvA6OAGcBLQHkl22jgxfD+S8CPwmq4EUB+OFQ3ARhlZm3D4oNRYVuV1AMSEYm5Bp6KpzPwgplBkBOecvfXzWwy8IyZnQcsAk4Pt38NOB6YB2wGfgLg7uvM7GZgcrjdOHdfV92BlYBERBoxd58P7FdJ+1rgqEraHRhTxb4eAR6p6bGVgHaRF5dQumLVzjeUxqWsNOoIJF15+s4FpwQkIhJj6TwZqRKQiEjMpWsCUhWciIhEQj0gEZEY04qoIiISGVcCEhGRKGhJbhERSTpP4zJsFSGIiEgk1AMSEYk5nQMSEZEIqApOREQioh6QiIgkXTpPxaMiBBERiYR6QCIiceZBKXY6UgISEYk5XYgqIiJJ56RvEYLOAYmISCTUAxIRiTVdByQiIhFREYKIiEQiXc8BKQGJiMSYe/omIBUhiIhIJNQDEhGJORUhiIhIJFSEICIikUjXc0BKQCIiMeZY2iYgFSGIiEgk1AMSEYm5ND0FpAQUR5fftoDhR+aRtzabC0cNBODQ49dxzuXL6LFnIZeeNIC505sDsNd+G7n0dwsBMIMn7+7G/ya0jSjyhpXdtIw7np9HdtMyMjPhvVdb87c7unDSj1fz/Z+tpmvvrZw2cCAb1qf/r/UVdy5m+NEF5K3J4oIj+wFw6Il5nHvlCnr0LWLs8X2ZO61ZxFEmR2WfxY9+uZyRx2zAHfLWZHH7ZT1ZtzI74khrSdcBNQwzu8nMrjKzcWZ2dJSxxMnEZzvwq9F77dC28Ktcbr5gT2Z83HKH9kVzcrnke/sw5viB/Gr0Xoz97UIyMtPz+1JxkXH16Xtw0Xf7c9Gofgw7vID+Qzcxc3Jzrj1zD1YsSdE/MLXwxtPtuP7s3ju0LZydw7if9WL6R80jiioalX0Wzz3QiYuO7sfF3+3Hx2+24pzLV0YUXT3xOtxiLBZfFd39hobat5kZYO5e1lDHqG8zPmlJ5+5FO7QtmZdb6bZFhZnb7mc39bQt1wwYhZuD95uV5WRmB+/365mN45t+ohkft6Bz9607tC2ZlxNRNNGq7LPYvHH7/4uc3LI0/3+RupLeAzKz683sKzN7H+gXtj1mZqeG939vZrPMbJqZ3R62fc/MPjazz8zsTTPrHLZ3NLOJZjbTzB4ys0Vm1sHMepnZHDN7ApgB9DCzX5rZ5HC/v0mI5xwz+8TMPjezv5hZ5reCjrl+gzfyl4nTeXDCDO67vhdlpenZXQfIyHDuf2M2T0+bwWeTWjLns8b1bV9q7sfXLOfJKbM48gd5PHHbblGHUyfuVutbnCU1AZnZ/sCZwGDgeOCACs+3B74P7OPug4BbwqfeB0a4+xDgn8DVYfuNwNvuvg/wHNAzYXd9gfvD5/qFjw8Mj72/mR1mZnsDZwAHu/tgoBQ4u5K4zzezKWY2pdgL6/ox1Ls5n7fggu/uy9iTBnDGxcvJbpoynb1dVlZmXDyqP2cPG0C/IZvZvd+WqEOSmHrsD104Z9gA3v5XG0766Zqow6kT99rf4izZPaBDgRfcfbO7bwBeqvB8PlAIPGxmPwA2h+3dgQlmNh34JbBP2H4IQULC3V8H1ifsa5G7fxTeHxXePgM+BfoTJKSjgP2ByWb2efi4T8Wg3X28uw9z92HZFt9hjiXzctmyOYNee6X/H+VNG7L44oMWHHB4QdShSMy9/UJbDjk+P+owaq18RVT1gBqYu5cQ9FKeA04EXg+fug/4k7vvC1wA1CQLbEq4b8Dv3H1weNvT3R8O2x9PaO/n7jfV1/tJhs49irYVHXTqVkSPPQpZubRJxFE1jNbtSmjeqgSAJjllDD2sgCVfN404Komjrr23n0MdeUw+S+al8O+JA261v8VYsosQJgGPmdnvwmN/D/hL+ZNm1gJo5u6vmdkHwPzwqdbAsvD+6IT9fQCcDvzBzEYBVdUfTwBuNrO/u/tGM+sGFANvAS+a2V3uvsrM2gEt3X1RvbzbWrr23q8ZNLKAVm1L+NtHn/PkXd0oyMviot8sonW7EsY9+hXzZzXj+h/1Y+CwAk6/eDklxcG3nT/9anc2rE/ParB2nYu56u7FZGQ4GRkw6eU2fPxma07+6WpOu3gV7ToW8+Cbs/nk7Vbc/cueO99hCrv2/kUMGrmR1u1KeHLKLP52R2cK1mdx8S3LaN2+hJv/toCvZ+Zw/Vl7RB1qg6vsszjwyAK671FEWRmsWtaEe6/pHnWYUgnzJA8Smtn1BElkFbCYYEhsIPAKQUJ5kaCHY8Dt7v64mZ0M3EUwxPY2cIC7H25mnYB/AJ2BDwl6Tb2ALsAr7j4w4biXAj8LH24EznH3r83sDOA6gt5gMTAmYejuW1pltPcR2cfWx0eR8rykOOoQ4iPug+0SiTf9uanuPqwu+2jap5t3u3VMrV+/4Kzr6xxDQ0l6Gba73wrcWs0mB1bymhcJElNF+cAx7l5iZiMJElMRsJAgqSXu4x7gnkr2/TTwdI3fgIhIsiXh+01YATwFWObuJ5pZb4Jz7O2BqcC57r7VzJoCTxCcP18LnOHuC8N9XAecR1DQNdbdJ1R3zFidA6qFngQFBF8A9wI/jzgeEZF6VvsChF0sQrgU+DLh8R+Au9x9T4LRp/PC9vOA9WH7XeF2mNkAgirnfYBjgft3dllLSicgd5/r7kPcfT93P8DdJ0cdk4hIvWvgmRDMrDtwAvBQ+NiAIwkKwgAeB04J758cPiZ8/qhw+5OBf7p7kbsvAOZRyYhWopROQCIiUi/uJri+svwiwvZAXliZDLAU6Bbe7wYsgW2Vy/nh9tvaK3lNpZSARETizOt8HVCH8gvpw9v5ibs3sxOBVe4+NdlvLRZzwYmISDXqVoSwZidVcAcDJ5nZ8QQVyK0ICrbamFlW2MvpzvZLYZYBPYClZpZFcJnM2oT2comvqZR6QCIisWd1uFXP3a9z9+7u3ougiOBtdz8beAc4NdxsNNsrkV9i+/WYp4bbe9h+ppk1DSvo+gKfVHfsKntAZnYf1eRddx+7szcmIiIp6xrgn2Z2C8E0Zg+H7Q8DfzOzecA6gqSFu880s2eAWUAJwTWVpdUdoLohuCl1DF5EROpDkq5zdvd3gXfD+/Op/LrMQuC0Kl6/s+s8d1BlAnL3xxMfm1kzd99c1fYiItJA0nSijZ2eAzKzkWY2C5gdPt7PzO5v8MhERCStJyOtSRHC3cAxBFUOuPsXwGENGZSIiGzXqNcDcvclFZqqPbEkIiKyMzW5DmiJmR0EuJll8+35gkREpCHFvCdTWzVJQBcSXJTUDfiGYG2d2s8NLiIiuybm53Jqa6cJyN3XAGcnIRYREamEpWkPqCZVcH3M7GUzW21mq8zsRTPrk4zgREQkfdWkCOEp4BmCVUa7As8SrEIqIiINrS5LMcS851STBNTM3f/m7iXh7UmCCetERKTB1eEaoJifO6puLrh24d3/mNm1BEuzOnAG8FoSYhMREYh9T6a2qitCmErwtstT6AUJzzlwXUMFJSIiCRpbAnL33skMREREGpcaLUhnZgOBASSc+3H3JxoqKBERSdDYekDlzOxG4HCCBPQacBzwPqAEJCLS0MonI01DNamCOxU4Cljh7j8B9iNYglVERJLAvPa3OKvJENwWdy8zsxIzawWsYsd1v0VEpCHFPJHUVk0S0BQzawP8laAybiPwYYNGJSIiaa8mc8FdHN590MxeB1q5+7SGDUtERNJddReiDq3uOXf/tGFCEhGRRHE/l1Nb1fWA7qjmOQeOrOdYUoM7Xrw16igkZopOOCDqEGIj500NkGxTWE/7SdMquOouRD0imYGIiEjjUqMLUUVEJCIpMKt1bSkBiYjEnRKQiIhEIV2LEGqyIqqZ2TlmdkP4uKeZHdjwoYmICNCoF6S7HxgJ/L/wcQHw5waLSEREGoWaDMENd/ehZvYZgLuvN7MmDRyXiIiUi3lPprZqkoCKzSyT8CMws45AWYNGJSIiQGpMKlpbNUlA9wIvAJ3M7FaC2bF/1aBRiYjIdo3tQtRy7v53M5tKsCSDAae4+5cNHpmIiAQaaw/IzHoCm4GXE9vcfXFDBiYiIumtJkNwrxLkXyNYkrs3MAfYpwHjEhGRUKM9B+Tu+yY+DmfJvriKzUVEpL411gRUkbt/ambDGyIYERGpoDFXwZnZFQkPM4ChwDcNFpGIiDQKNekBtUy4X0JwTuj5hglHRES+pTH2gMILUFu6+1VJikdERCpK0wRU5VxwZpbl7qXAwUmMR0REKiifDaE2t53u2yzHzD4xsy/MbKaZ/SZs721mH5vZPDN7unwKNjNrGj6eFz7fK2Ff14Xtc8zsmJ0du7rJSD8Jf35uZi+Z2blm9oPy287floiIpIAi4Eh33w8YDBxrZiOAPwB3ufuewHrgvHD784D1Yftd4XaY2QDgTIJLdI4F7g9H0apUk9mwc4C1wJHAicD3wp8iIpLiPLAxfJgd3pzgb/5zYfvjwCnh/ZPDx4TPH2VmFrb/092L3H0BMA+odume6s4BdQor4Gaw/ULUbTHX5I2JiEg9aOC/uGFPZSqwJ8FyO18Dee5eEm6yFOgW3u8GLAFw9xIzywfah+0fJew28TWVqi4BZQIt2DHxlFMCEhFJhrpfB9TBzKYkPB7v7uN3OERwvn+wmbUhmHy6f52OWEPVJaDl7j4uGUGIiEg16paA1rj7sBodxj3PzN4hWIS0TViMVgJ0B5aFmy0DegBLzSwLaE1wmqa8vVziaypV3Tmg9Jz/W0Qk1TTgktxm1jHs+WBmucB3gS+BdwiW3wEYDbwY3n8pfEz4/Nvu7mH7mWGVXG+gL9uL2SpVXQ/oqJ2HLiIiKa4L8Hh4HigDeMbdXzGzWcA/zewW4DPg4XD7h4G/mdk8YB1B5RvuPtPMngFmEUxaMCYc2qtSlQnI3dfV8U2JiEgdGQ07F5y7TwOGVNI+n0qq2Ny9EDitin3dCtxa02Pv8mSkIiKSZGla9qUEJCISZ415NmyJ3hV3Lmb40QXkrcnigiP7AXDoiXmce+UKevQtYuzxfZk7rVnEUSZf81alXH77Enr1L8Qd7ryiB19ObR51WPXq6tGTGDloMXkFufzkph9ua//+kTP5/uGzKHXjo2k9+Mvzw9l/76Wc/8PJZGeWUVyawYPPDeez2V0ByMos5dKz/sfgfsvxMuOhfw9j0qe9o3pbdXb5H+Yz/Mg88tZmc+GxwZJlP7piKSO/u56yMiNvbRZ3XNWHdaua0KJVCZf/cQFddy9ka1EGd17dm0VfNb7/L3EUywRkZjcBG4FWwCR3fzPaiKL1xtPteOnRDvzyniXb2hbOzmHcz3ox9g9LI4wsWheNW8aUd1tyy/m9yMouo2lu+n1NfP1/fXnhnQH830//u61tcL9vOGS/RZw37gcUl2TSpuUWAPI35vB/941ibX5zenddxx8ve53Trj4LgHNO+Jy8glzO/dXpmDmtmhdF8n7qy8TnO/DyE5256o7529qeG9+FJ+7sDsDJP17B2WOXcd+venPmmG+YP6sZN1/Yl+59tjBm3CKuOycpl7nUn/T71QZqNhVPZNz9hsaefABmfNyCgvU7fldYMi+HpV/nRBRR9Jq1LGXfEZt4/al2AJQUZ7BpQ7XTTqWkaXO7ULCp6Q5tJx/+JU+9vh/FJcH7zSvIBWDekg6szQ96gAu+aUvTJqVkZwVFSMcf/BV/f20/ANyN/I2p/bsz45NWFOTt+H9i88bt//45uWW4B1eS9NxzC59/2AqApfNz6dy9iDYdipMXbH1owDLsKMWmB2Rm1xPUlq8imOZhqpk9Brzi7s+Z2e+BkwjK+95w96vMrDPwINAn3M1FBIvlveLuA8P9XgW0cPebzGwscGG4j1nufqaZfQe4J3y9A4e5e0ES3rLUwW49t5K/NpMr71pCn322MHdaMx74dVeKtqRfEqqoR+d89u27gvNOmcLW4kweeG44cxZ23GGb7wxdyNxF7SkuyaRFbtDb+ekpUxm813K+Wd2Ke54ayfqC9BuGGn3VEo7+/lo2FWRyzVlBL2f+l804+Jh1zJzckr3220jnbkV02G0reWuyI4625tL1HFAsekBmtj9BLflg4HjggArPtwe+D+zj7oOAW8Kn7gX+G87iOhSYuZNDXQsMCfdxYdh2FUG9+mDgUGBLJfGdb2ZTzGxKMak9dJEuMjOdPffdwitPtGfMqH4Ubs7gjF+sijqspMjMCIbQLv7dSTz43IHcdMFbJH7V7dV1Pef/8BPuePKQYPtMp1O7Tcyc15nzb/k+M+d34qLTqr0+MGU9fnsPzj14MO+82J7v/WglAM882JUWrUr586szOHn0Sr6e2Zyyaq9OiaE07QHFIgER/OF/wd03u/sGgitqE+UDhcDD4VIQm8P2I4EHIJjLyN3zd3KcacDfzewcgl4QwAfAnWHvqE3C5HvbuPt4dx/m7sOyaVrxaYnAmuXZrF6ezZzPgiGn919pzZ77fuu7Q1pavb45kz7tBRizF3airMxo3aIQgI5tN3HzxRP53SPf4ZvVwbBT/sambCnKYtJnvQB4d0pv+u6+Jprgk+TtF9tzyLHrgY2mYF8AAB4WSURBVGBo7s6r+zDmhIHcdkUfWrcvZsWS1B6CTBdxSUDVCpPCgQRTf58IvF7N5iXs+L4Sf9NOIJjpdSgwOZzn6PfAz4Bc4AMzS7Gzk43T+tXZrPmmCd33CP7wDj50I4vnNo4/Ku9/vjtD+i0HoHvnfLKzysjfmEOL3CJ+d8kExj9/ADO+3i3hFcaHX/RkcPia/ff+hkXftI0g8obVtVfhtvsjv7ueJfOD34fmLUvIyi4D4NgzVzP9k5Y7nC+Kvbr0fmLeA7JgCp+IgzAbCjwGDCc4L/Up8BdgIPAKQcJp5u6rzKw1MN/d25vZP4GP3P3ucBqJFgS9o+VAP4JKuv+Grx8H9HT3hWaWDSwCBgDt3f3rMI7ngCfd/d9VxdrK2vlwS+4sRdfev4hBIzfSul0J61dn87c7OlOwPouLb1lG6/YlbNqQydczc7j+rD2SGlfU+uyzhctvX0JWtrNicRPuuLwHG/OjOa1ZdMIBO9+oFn7987cZvNdyWrcoZH1BLo++tD9vfLgn1/x4Env2WEdxSQYPhOXW557wGWcd9wXLVrXa9vqr7jqOvIJcOrcr4P/O+y8tmhWRV5DLHx47jFXrWjRIzDlvTmuQ/Sa69p55DBpRQKu2Jaxfk8WTd3fngMPz6N4nKMlfuawp913fi7Urm7D3kAKuvGM+uLHoq1zuuqY3Gzck5/dkYuHfp9Z0ItCq5O7Ww/f40RW1fv3M266ocwwNJRYJCL5VhLCYIAmVJ6APCCbCyyGYmeJ2d388LEIYT1CEUApc5O4fhsNplxLMxDofWEgwPcQ7BDO3GkGi+b2Z3QccAZQRnEP6sbtXeaInigQk8ddQCSgVJSMBpYp6S0Dn1iEB3R7fBBSbKrgazCFU2ZxEKwlW4avYfi9BgUJFh1Sy7SW7EKaISNKpCk5ERKQexaYHJCIiVUjTHpASkIhInKVANVttKQGJiMSYkb7LU+sckIiIREI9IBGRuNMQnIiIRCFdy7CVgERE4k4JSEREIpGmCUhFCCIiEgn1gERE4sx1DkhERKKiBCQiIlFQD0hERKKRpglIRQgiIhIJ9YBERGJOQ3AiIpJ8mg1bREQik6YJSOeAREQkEuoBiYjEmKFzQCIiEhUlIBERiYJ5emYgJSARkThL4yo4FSGIiEgk1AMSEYk5FSGIiEg00jQBaQhORCTmzGt/2+m+zXqY2TtmNsvMZprZpWF7OzObaGZzw59tw3Yzs3vNbJ6ZTTOzoQn7Gh1uP9fMRu/s2EpAIiKNWwlwpbsPAEYAY8xsAHAt8Ja79wXeCh8DHAf0DW/nAw9AkLCAG4HhwIHAjeVJqyoagttVZlh2k6ijiAUv3hp1CLGRO2l21CHExn8WfBx1CLGR2aWedtSAQ3DuvhxYHt4vMLMvgW7AycDh4WaPA+8C14TtT7i7Ax+ZWRsz6xJuO9Hd1wGY2UTgWOAfVR1bCUhEJM6SuCS3mfUChgAfA53D5ASwAugc3u8GLEl42dKwrar2KikBiYjEXd0SUAczm5LweLy7j6+4kZm1AJ4HLnP3DWa2/fDublb/aVAJSEQkxuphLrg17j6s2mOYZRMkn7+7+7/C5pVm1sXdl4dDbKvC9mVAj4SXdw/blrF9yK68/d3qjqsiBBGRRsyCrs7DwJfufmfCUy8B5ZVso4EXE9p/FFbDjQDyw6G6CcAoM2sbFh+MCtuqpB6QiEjcNexccAcD5wLTzezzsO3/gN8Dz5jZecAi4PTwudeA44F5wGbgJ0GIvs7MbgYmh9uNKy9IqIoSkIhIzDVkEYK7v08w0leZoyrZ3oExVezrEeCRmh5bCUhEJM7SeDJSJSARkZizsqgjaBgqQhARkUioByQiEncaghMRkShoOQYREUk+p6HLsCOjc0AiIhIJ9YBERGJOQ3AiIhINJSAREUm2epiMNLaUgERE4sxdRQgiIiL1ST0gEZGY0xCciIhEQwlIRESioB6QiIgknwNl6ZmBVIQgIiKRUA9IRCTu0rMDpAQkIhJ3OgckIiLR0IWoIiIi9Uc9IBGRmNMQnCTN5bctYPiReeStzebCUQMBOPT4dZxz+TJ67FnIpScNYO705tu2791/M2N/t5BmLUopKzPGnjSA4qL07tx27LqVX96zmDYdS8DhtSfb8++HO0YdVoPqsFsRV/3xK9q234q78Z9nOvPiE904+xeLOPb0leSvywbg8Tt3Z/Kkduy1bwFjb54HgJnz9/t68r83O0T5FupsY34md13Vg4WzczCDK+5cTNPcMu67tgdbNmXQuftWrvnzIpq3LGP2Z82455c9gOAc/rlXruDg4/IB+Nf4jvznqXaYQe/+hVx512Ka5MT0r7yjIoS4MLPDgavc/cRqthkMdHX313Zx3++G+55SpyDraOKzHXj58U5cdeeCbW0Lv8rl5gv2ZOxvF+2wbUamc/Xd8/nj5X1Y8GUzWrYpobTYkh1y0pWWGOPHdWXe9GbkNi/lT69/xaeTWrJ4bk7UoTWY0lLjr7/vzdezWpDbvIR7n/+czz5oC8C/H+vK849032H7RXObMfaHgykrNdp23Mr9L37GR++0p6w0dX8/HrihG8MO38Cv/7qQ4q1G0ZYMrjtzD35+wzIGjdzEhH+047kHOjH66hX06reFP70+h8wsWLsyi4uO7seI7+azfnU2/364A399dzZNc51bLtidd19sy6gz1kX99ioVzIadnhkoll+TzayuiXEwcHx9xBKFGZ+0pCBvx49gybxcls7P/da2+x+Wz4LZuSz4shkABXlZlJWl7h+Ymlq3Kpt504P3vGVTJkvm5dChS3HEUTWs9aub8PWsFgBs2ZTFkvnNaN+5qMrtiwoztyWbJk3LUv489qYNGUz/qDnHnhUkiuwmTovWpSyd35R9R2wCYMhhBbz/ahsAcpo5meF/o+KiDCzhv0VpiVFUmEFpCRRtyaB955j/7pTV4RZjDdYDMrNewCvuPjB8fBXQAjgc+Bg4AmgDnOfu75nZj4EfhNtkmtnxwH3AQCAbuMndX6xwjAOBe4AcYAvwE2ABMA7INbNDgN8Br1S2LzPLBR4F9gNmA9/+Cx9z3XoX4m7c+sQcWrcv4d2X2vHcX7pEHVZSde6+lT0GbmH2p82iDiVpOnUrZI+9NzHni5YMGLqB7529nKNOWcXcGS346+/7sHFD8F+736ACLv/tXDp1LeT2q/dK6d7PisVNad2+hDsu78n8mTn0HbSFi25exu57FfLh66056Lh83nulDau/yd72mtmfNuOOK3qwamkTrr5vMZlZ0KFLMadetIpzDxhA0xxn6Hc2sP/hBRG+s8Yrqh5QlrsfCFwG3JjQPhQ41d2/A1wPvB1udwRwm5k1r7Cf2cCh7j4EuAH4rbtvDe8/7e6D3f3pavZ1EbDZ3fcO49i/smDN7Hwzm2JmU4q9sH4+gXqSmeXsc0ABf7i0D1f+sD8HH7uewQdviDqspMlpVsqvH1rIgzd0ZfPGzKjDSYqcZqX86t4v+ctve7N5Uxav/qMLP/3uMMacPIR1q5rw82vnb9t2zrSWXHjiUC49dTCnX7CU7CYx/0pcjdJSmDe9GSf+aA33T/yKnGZlPP2nTlxx52Jefrw9Y47Ziy0bM8hqsr2r13/oZv767hzu+89X/PO+TmwtNAryMvlwQmse/3gWT302g8LNmbz1fNsI39nOmXutb3EWVQL6V/hzKtAroX2iu5cPxI4CrjWzz4F3CXo5PSvspzXwrJnNAO4C9qnieFXt6zDgSQB3nwZMq+zF7j7e3Ye5+7Bsi9c5hjXLmzD945ZsWJ9NUWEmk99pw54DN0UdVlJkZjm/fmghb/+rLR/8p03U4SRFZlYZv7r3S955uRP/mxgUFOStbUJZmQWFCc/uxl77bvzW65bMb8aWzZn02it1fzc6dCmmY5di+g/dDMAhJ+Yxb3ouPfsW8bt/zufPE77i8FPy6LL7t4cle/YtIrd5GQvn5PDZey3YrcdW2rQvJSsbDj4+j1lTKn63jRGv4y3GGjIBlVTYf+Jf7vLfkFJ2HAZM/N9hwA/DXsxgd+/p7l9WOMbNwDvhMN/3KhwjUU32lZKm/rc1vftvoWlOKRmZzr7DC1g8N+VGEmvBueKOJSyZm8O/xqd39dt2zmW3zmXJ/Ga88Fi3ba1tO27ddv+go9eyaG4wFNm5eyEZmcFfoE5dC+nRZwsrl8XrC9SuaNephA5dt7JkXlMAPn+vJT37FpG3JvgTUlYGT93TmRPPXQvAisVNKC0JXrtyaTZL5uXQuftWOnUr5stPm1G42XCHz99vSc894zWysSPfvipqbW4x1pBVcCuBTmbWHtgInAi8vguvnwBcYmaXuLub2RB3/6zCNq2BZeH9Hye0FwAta7CvScBZwNtmNhAYtAvxNZhr7/2aQSMLaNW2hL999DlP3tWNgrwsLvrNIlq3K2Hco18xf1Yzrv9RPzZuyOJfD3Xm3pdn4W5Mfqc1n7yd/r2BfQ7cxNGnrWf+rBzunzgHgEd/14XJb7eKOLKGs8/+Gzj6lNUsmNOMP/07+K/w+J27850TV9Onf/DdbeWyHO69Yc9t25/+86WUlBheBn++aQ82rM+ucv+pYMwty/jDL3anpNjYredWrrxrMW8+15aXHwt6gwcfl8+oM4NBlBmfNOfpP/UmKwsyMpxLfruU1u1Lad1+M4eekM+YY/qRmeXsOXALx52zNsq3tVPpeh2QeQNmSDMbC1xKkCTmAwsJihCucvcpZtYBmOLuvcIihGHu/ovwtbnA3cBBBD2pBe5+YmIZtpmNBB4n6Dm9CpwT7qsdQdLJJihCeKmKfSUWIXwJdAPGVFeG3SqjvY/IPra+PqKU5sVbd75RI5HRsuXON2ok/jPnvahDiI3MLvOmuvuwuuyjVctufuDQMbV+/VuTrq9zDA2lQRNQOlIC2k4JaDsloO2UgLartwQ05OJav/6t934V2wSUcheiiog0Kg6WusWL1VICEhGJuzQdqYrlTAgiIpL+1AMSEYm79OwAKQGJiMRd3Gc0qC0lIBGRuEvTBKRzQCIiceY06GzYZvaIma0KpzQrb2tnZhPNbG74s23YbmZ2r5nNM7NpZjY04TWjw+3nmtnomrw1JSARkcbtMaDixY3XAm+5e1/grfAxwHFA3/B2PvAABAmLYELn4cCBwI3lSas6SkAiIjFm1H4m7JqcO3L3SUDF1fhOJphlhvDnKQntT3jgI6CNmXUBjiGcTNrd1wMT+XZS+xadAxIRibvknwPq7O7Lw/srgM7h/W7AkoTtloZtVbVXSwlIRCTu6paAOphZ4vyW4919fM0P7W7WMNOhKgGJiMRZeRFC7a2pxVxwK82si7svD4fYVoXty4AeCdt1D9uWEUw0ndj+7s4OonNAIiJS0UtAeSXbaODFhPYfhdVwI4D8cKhuAjDKzNqGxQejwrZqqQckIhJzDXkhqpn9g6D30sHMlhJUs/0eeMbMzgMWAaeHm78GHA/MAzYDPwFw93VmdjMwOdxuXMLq1lVSAhIRibsGTEDu/v+qeOqoSrZ1oNLFidz9EeCRXTm2EpCISKzFf2nt2tI5IBERiYR6QCIiceakbQ9ICUhEJO60IqqIiERByzGIiEg00jQBqQhBREQioR6QiEicOVCWnj0gJSARkVhL3+uAlIBEROJOCUhERCKRpglIRQgiIhIJ9YBEROJMRQhSrsDXrZm49alFUccBdADWRB1ETET/WWyI9OiJIv8sMrtEefQdRP5ZALvXfRcOnp5TISgB7SJ37xh1DABmNqUWqxymJX0W2+mz2C6tPgudAxIREak/6gGJiMSZzgFJDI2POoAY0WexnT6L7dLns0jTITgloBTl7unzn6uO9Flsp89iu7T6LJSAREQk+dJ3Kh4VIYiISCTUAxIRiTMHynQdkMSAmTVz981RxxElMzPgbKCPu48zs57Abu7+ScShJZ2ZNQe2uHuZme0F9Af+4+7FEYeWVGY2tLrn3f3TZMXSINJ0CE4JKEWY2UHAQ0ALoKeZ7Qdc4O4XRxtZJO4HyoAjgXFAAfA8cECUQUVkEnCombUF3gAmA2cQJOjG5I7wZw4wDPgCMGAQMAUYGVFc9SNNE5DOAaWOu4BjgLUA7v4FcFikEUVnuLuPAQoB3H090CTakCJjYY/4B8D97n4asE/EMSWdux/h7kcAy4Gh7j7M3fcHhgDLoo2urjy4Dqi2txhTAkoh7r6kQlNpJIFEr9jMMglGxzGzjgQ9osbIzGwkQY/n1bAtM8J4otbP3aeXP3D3GcDeEcYj1dAQXOpYEg7DuZllA5cCX0YcU1TuBV4AOpvZrcCpwK+iDSkylwHXAS+4+0wz6wO8E3FMUZpmZg8BT4aPzwamRRhP3Tl4mk5Gap6mY4vpxsw6APcARxOMbb8BXOruayMNLCJm1h84iuCzeMvdG2sy3sbMMoAW7h6fubmTzMxygIvYPjw9CXjA3Quji6puWmd19JGtTqn16yesf2hqXCdlVQ8oRbj7GhrfieXqdAA2u/ujZtbRzHq7+4Kog0o2M3sKuJBgOHYy0MrM7nH326KNLBruXmhmDwKvufucqOOpN2naUdA5oBRhZnuZ2VtmNiN8PMjMGuWwk5ndCFxDMPQEkM32IZfGZkDY4zkF+A/QGzg32pCiY2YnAZ8Dr4ePB5vZS9FGJVVRAkodfyX4g1sM4O7TgDMjjSg63wdOAjYBuPs3QMtII4pOdnhO8BTgpfD6n/T8ulwzNwIHAnkA7v45QVJOXe7Bhai1vcWYElDqaFbJhZYlkUQSva0enLwsr4JrHnE8UfoLsBBoDkwys92J0/qsyVfs7vkV2lI/IbvX/hZjOgeUOtaY2R5s/6N7KsE1D43RM2b2F6CNmf0c+ClBD7HRcfd7CaoCyy0ysyOiiicGZprZWUCmmfUFxgL/izimOvOY92RqSwkodYwhWN+kv5ktAxbQCIsSwml4niaYcmYD0A+4wd0nRhpYRMzshiqeGpfUQOLjEuB6oAh4CpgA3BxpRHUW/55MbSkBpYDwosuL3f3ocLgpw90Loo4rCu7uZvaau+8LNMqkU8GmhPs5wIk03uvDAE5w9+sJkhAAZnYa8Gx0IUlVlIBSgLuXmtkh4f1NO9u+EfjUzA5w98lRBxI1d78j8bGZ3U7wrb+xuo5vJ5vK2lKHluSWGPgsLCd9loRvve7+r+hCisxw4GwzW0TwWRhB52hQtGHFQjOge9RBJJuZHQccD3Qzs8RzYq1Ih2KdNJ0JQQkodeQQTER6ZEKbA40xAR0TdQBxYWbT2V7llQl0pHGe//mGYNbrk4CpCe0FwOWRRFRPHHD1gCRK7v6TqGOIkcrOfzXKc2IE53zKlQAr3T31v/HvonB2+C/M7KnGthZSKlMCShEVhhXK5QNT3P3FZMcTsU+BHsB6guG3NsAKM1sJ/Nzdp1b34jTTBZhZXpRiZi3NbIC7fxxxXFHpZWa/AwYQjBoA4O59ogupjtzTdghOF6KmjhxgMDA3vA0iGOs/z8zujjKwCEwEjnf3Du7eHjgOeAW4mGCxusbkAWBjwuNNYVtj9SjB+y8BjgCeIA2mafIyr/UtztQDSh2DgIPdvRTAzB4A3gMOAaZX98I0NMLdf17+wN3fMLPb3f0CM2saZWARME+Y0j5cmrsx/7/Odfe3zMzcfRFwk5lNBaq6Xio1pGkPqDH/oqaatgTLcZdPM9IcaBeWaBdFF1YklpvZNcA/w8dnACvD66XS839q1eab2Vi293ouBuZHGE/UisJlKeaa2S8IVkNtEXFMdVLA+glv+nMd6rCLNfUWTD3TekApwszOI1h07V2C8x6HAb8F/gHc5O6/jC665ArXRrqRoPcH8AHwG4Lk3NPd50UVW7KZWSeCqXiOJCiYegu4zN1XRRpYRMzsAIILcdsQzIDQGviju38UaWBSKSWgFGJmXQmm2v+S4FvdUnefFG1UIiK1oyG4FGFmPyNYhrs7wXonI4AP2fG6oEbBzDoCVwP7sGOlU2P8LP4I3AJsIVgDZxBwubun/In3XWFmL1PNrNfuflISw5EaUgJKHZcCBwAfufsR4ZLUv404pqj8nWBC0hMJVgMdDayONKLojHL3q83s+wTLMvyAYBnqRpWAgNujDkB2nRJQ6igMlxvGzJq6+2wz6xd1UBFp7+4Pm9ml7v5f4L9m1ljnhSv/P3wC8Ky75wcThjcu4e8BAGaWS3AuMH2W5E5Tug4odSw1szbAv4GJZvYisCjimKJSfqX7cjM7wcyGAO2iDChCr5jZbGB/4K1weLIw4pgiY2bfQ0typwwVIaQgM/sOQXXP6+6+Nep4ks3MTiS4BqoHcB/BhJO/cfdG+YfGzNoB+WFJfjOglbuviDquKITX/BwJvOvuQ8K26eHyHRIzGoJLQYnDDY2Ru78S3s0nuNq90TGzI939bTP7QUJb4iaNcZJaCJfkrvBZ6Ft2TGkITlKOmf3RzFqZWbaZvWVmq83snKjjSrLDwp/fIyjGqPizsdphSW4zu480WJI7XSkBSSoa5e4bCP7QLgT2BBrNhbihAjO7ApiRcJtJMC3TjCgDi9glBOX55Uty5wOXRRqRVElDcJKKVPm1fXqZfgTl+S8SzJDxPeCTqIKKUjgV06vufgQJS3JLfCkBSSoqr/zaAlzUGCu/3P03AGY2CRiasBzDTcCrEYYWmbAIo8zMWrt7/s5fIVFTFZykJFV+BcxsDjDI3YvCx02Bae7eKK8RCy9PGEKwZEfi0vVjIwtKqqQekKQMVX5V6gngEzN7IXx8CvBYdOFE7l80zt+DlKQekKQMM7vJ3W8ys0cJSmst8ae7/zTSACNiZkOBQ8OHk9z9syjjEakpJSBJGWZ2Jd9OPIT3cfc7IwpNYsLMDgZuAnYnGOEp/3KSuktypzENwUkqUeWX7MzDwOXAVKA04lhkJ9QDkpQTVn6dkFD51ZKg/Paw6l8p6c7MPnb34VHHITWjHpCkos5A4hx4W8M2kXfM7DaCQoRtS9W7+6fRhSRVUQKSVKTKL6lKee9n//Bn+fnCRrdYYSpQApKU4+63mtl/2F759RNVfkno3UradJ4hppSAJCWFQyoaVpGKNibczyGYL/DLiGKRnVARgoikrXBmiAnufnjUsci3aTZsEUlnzYDuUQchldMQnIikDTObzvZzPplAR2BcdBFJdTQEJyJpw8x2T3hYAqx095Ko4pHqKQGJiEgkdA5IREQioQQkIiKRUAKSlGRmpWb2uZnNMLNnw0Xparuvx8zs1PD+Q2Y2oJptDzezg2pxjIVm1qGm7RW22Vjd85Vsf5OZXbWrMYokmxKQpKot7j7Y3QcSzAV3YeKTZlarCk93/5m7z6pmk8OBXU5AIvJtSkCSDt4D9gx7J++Z2UvALDPLNLPbzGyymU0zswsALPAnM5tjZm8Cncp3ZGbvmtmw8P6xZvapmX1hZm+ZWS+CRHd52Ps61Mw6mtnz4TEmh+vRYGbtzewNM5tpZg+xfe2iKpnZv81savia8ys8d1fY/paZdQzb9jCz18PXvGdm/evjwxRJFl0HJCkt7OkcB7weNg0FBrr7gvCPeL67HxBeEf+Bmb0BDCFYU2gAwSzas4BHKuy3I/BX4LBwX+3cfZ2ZPQhsdPfbw+2eAu5y9/fNrCcwAdgbuBF4393HmdkJwHk1eDs/DY+RC0w2s+fdfS3QHJji7peb2Q3hvn8BjAcudPe5ZjYcuB9NuikpRAlIUlWumX0e3n+PYCGyg4BP3H1B2D4KGFR+fgdoDfQFDgP+4e6lwDdm9nYl+x9BsLz1AgB3X1dFHEcDA8y2dXBamVmL8Bg/CF/7qpmtr8F7Gmtm3w/v9whjXQuUAU+H7U8C/wqPcRDwbMKxm9bgGCKxoQQkqWqLuw9ObAj/EG9KbAIucfcJFbY7vh7jyABGuHthJbHUmJkdTpDMRrr7ZjN7l2Ayzcp4eNy8ip+BSCrROSBJZxOAi8wsG8DM9jKz5sAk4IzwHFEX4IhKXvsRcJiZ9Q5f2y5sLwBaJmz3BnBJ+QMzK08Ik4CzwrbjgLY7ibU1sD5MPv0JemDlMoDyXtxZBEN7G4AFZnZaeAwzs/12cgyRWFECknT2EMH5nU/NbAbwF4Je/wvA3PC5J4APK77Q3VcD5xMMd33B9iGwl4HvlxchAGOBYWGRwyy2V+P9hiCBzSQYilu8k1hfB7LM7Evg9wQJsNwm4MDwPRzJ9rnNzgbOC+ObCZxcg89EJDY0FY+IiERCPSAREYmEEpCIiERCCUhERCKhBCQiIpFQAhIRkUgoAYmISCSUgEREJBJKQCIiEon/D2J2xPLqjjErAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x432 with 2 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "headline_train, body_train, labels_train, headline_val, body_val, labels_val = get_dataset_splitted()"
      ],
      "metadata": {
        "id": "8xHk15G08C45"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(labels_val)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PvqU0CWS8D2n",
        "outputId": "f54983e8-1f18-48be-d028-63086058b592"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[2 3 2 ... 3 3 2]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## **TEST**"
      ],
      "metadata": {
        "id": "BOkIhuwnqRY9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "###########   VAL\n",
        "unrelated_num = sum(map(lambda x : x == 3, labels_val))\n",
        "print(unrelated_num/len(labels_val))\n",
        "\n",
        "agree = sum(map(lambda x : x == 0, labels_val)) # 7.4, 1.7, 17.8 73.1\n",
        "print(agree/len(labels_val)) \n",
        "\n",
        "disagree = sum(map(lambda x : x == 1, labels_val))\n",
        "print(disagree/len(labels_val))\n",
        "\n",
        "discuss = sum(map(lambda x : x == 2, labels_val))\n",
        "print(discuss/len(labels_val))"
      ],
      "metadata": {
        "id": "z3JU9mYtCAEv",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "245d07f2-d951-4e74-b409-a894da21ffd3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.7312656328164082\n",
            "0.07363681840920461\n",
            "0.01680840420210105\n",
            "0.17828914457228615\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "preprocessing_layer = hub.KerasLayer(BERT_PREPROCESS_MODEL, name='preprocessing')\n",
        "encoder = hub.KerasLayer(BERT_MODEL, trainable=False, name='BERT_encoder')"
      ],
      "metadata": {
        "id": "5sXCrOmw9nqy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "text_test = ['NLP is such an amazing topic!', 'yessss man that\\'s right']\n",
        "text_preprocessed = preprocessing_layer(text_test)\n",
        "encoder_output = encoder(text_preprocessed)\n",
        "\n",
        "print(f'Encoder Outputs Len:{len(encoder_output[\"encoder_outputs\"])}')\n",
        "print(f'First encoder output Shape:{encoder_output[\"encoder_outputs\"][0].shape}')\n",
        "print(f'First encoder output Values:{encoder_output[\"encoder_outputs\"][0][0, :12]}')\n",
        "\n",
        "print(f'Last encoder output Shape:{encoder_output[\"encoder_outputs\"][-1].shape}')\n",
        "print(f'Last encoder output Values:{encoder_output[\"encoder_outputs\"][-1][0, :12]}')\n",
        "\n",
        "con = layers.Concatenate(axis=2)(encoder_output[\"encoder_outputs\"][-4:])\n",
        "print(con.shape)"
      ],
      "metadata": {
        "id": "QGVx_lvA-g65",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cd7d9c19-91cb-4016-8198-01401b4ad7bc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Encoder Outputs Len:12\n",
            "First encoder output Shape:(2, 128, 768)\n",
            "First encoder output Values:[[ 0.63196665  0.72824425  0.02815381 ...  0.40477645 -0.27801156\n",
            "   0.01448902]\n",
            " [ 0.19626704 -0.08015473  1.0844378  ... -0.24407747  0.0690643\n",
            "   1.4097763 ]\n",
            " [-0.5235941   0.9797519   0.7515236  ... -0.47939897 -0.2512071\n",
            "   1.2809124 ]\n",
            " ...\n",
            " [ 0.4516008  -1.3139483  -0.6335703  ...  0.076499   -0.57104117\n",
            "  -0.3675774 ]\n",
            " [ 0.05883533 -0.25499856 -0.05418177 ...  0.4626816  -0.5195377\n",
            "   0.46779627]\n",
            " [-0.2553115  -0.07954732  0.316671   ...  0.51137197  0.0100887\n",
            "   0.22091766]]\n",
            "Last encoder output Shape:(2, 128, 768)\n",
            "Last encoder output Values:[[-0.33739993 -1.1926047   1.9643435  ...  0.70534587 -0.7244281\n",
            "   1.4115175 ]\n",
            " [-0.4174343  -1.3315657   1.9003549  ...  0.5480451  -0.7924222\n",
            "   1.8019047 ]\n",
            " [-0.4441178  -1.3127457   1.9798666  ...  0.5427514  -0.7651248\n",
            "   1.8425766 ]\n",
            " ...\n",
            " [-1.5991316  -0.6604965  -0.13147555 ...  0.6915815  -0.8395759\n",
            "   1.68677   ]\n",
            " [-1.9298526  -0.46980605  0.02826781 ...  0.85070556 -0.9143539\n",
            "   1.6595093 ]\n",
            " [-2.0592198  -0.40460685  0.16588315 ...  0.85463023 -0.93360424\n",
            "   1.6381936 ]]\n",
            "(2, 128, 3072)\n"
          ]
        }
      ]
    }
  ]
}