第4章-文本分类.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "81a6f12e",
   "metadata": {},
   "source": [
    "下面几段代码展示朴素贝叶斯模型的训练和预测。这里使用的数据集为本书自制的Books数据集，包含约1万本图书的标题，分为3种主题。首先是预处理，针对文本分类的预处理主要包含以下步骤：\n",
    "\n",
    "- 通常可以将英文文本全部转换为小写，或者将中文内容全部转换为简体，等等，这一般不会改变文本内容。\n",
    "- 去除标点。英文中的标点符号和单词之间没有空格（如——“Hi, there!”），如果不去除标点，“Hi,”和“there!”会被识别为不同于“Hi”和“there”的两个词，这显然是不合理的。对于中文，移除标点一般也不会影响文本的内容。\n",
    "- 分词。中文汉字之间没有空格分隔，中文分词有时比英文分词更加困难，此处不再赘述。\n",
    "- 去除停用词（如“I”、“is”、“的”等）。这些词往往大量出现但没有具体含义。\n",
    "- 建立词表。通常会忽略语料库中频率非常低的词。\n",
    "- 将词转换为词表索引（ID），便于机器学习模型使用。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5936ceb0",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train size = 8627 , test size = 2157\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8627/8627 [00:48<00:00, 179.28it/s]\n",
      "100%|██████████| 2157/2157 [00:11<00:00, 180.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['python', '编程', '入门', '教程']\n",
      "{'计算机类': 0, '艺术传媒类': 1, '经管类': 2}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import os\n",
    "import requests\n",
    "import re\n",
    "from tqdm import tqdm\n",
    "from collections import defaultdict\n",
    "from string import punctuation\n",
    "import spacy\n",
    "from spacy.lang.zh.stop_words import STOP_WORDS\n",
    "nlp = spacy.load('zh_core_web_sm')\n",
    "\n",
    "\n",
    "class BooksDataset:\n",
    "    def __init__(self):\n",
    "        train_file, test_file = 'train.jsonl', 'test.jsonl'\n",
    "\n",
    "        # 下载数据为JSON格式，转化为Python对象\n",
    "        def read_file(file_name):\n",
    "            with open(file_name, 'r', encoding='utf-8') as fin:\n",
    "                json_list = list(fin)\n",
    "            data_split = []\n",
    "            for json_str in json_list:\n",
    "                data_split.append(json.loads(json_str))\n",
    "            return data_split\n",
    "\n",
    "        self.train_data, self.test_data = read_file(train_file),\\\n",
    "            read_file(test_file)\n",
    "        print('train size =', len(self.train_data), \n",
    "              ', test size =', len(self.test_data))\n",
    "        \n",
    "        # 建立文本标签和数字标签的映射\n",
    "        self.label2id, self.id2label = {}, {}\n",
    "        for data_split in [self.train_data, self.test_data]:\n",
    "            for data in data_split:\n",
    "                txt = data['class']\n",
    "                if txt not in self.label2id:\n",
    "                    idx = len(self.label2id)\n",
    "                    self.label2id[txt] = idx\n",
    "                    self.id2label[idx] = txt\n",
    "                label_id = self.label2id[txt]\n",
    "                data['label'] = label_id\n",
    "\n",
    "    def tokenize(self, attr='book'):\n",
    "        # 使用以下两行命令安装spacy用于中文分词\n",
    "        # pip install -U spacy\n",
    "        # python -m spacy download zh_core_web_sm\n",
    "        # 去除文本中的符号和停用词\n",
    "        for data_split in [self.train_data, self.test_data]:\n",
    "            for data in tqdm(data_split):\n",
    "                # 转为小写\n",
    "                text = data[attr].lower()\n",
    "                # 符号替换为空\n",
    "                tokens = [t.text for t in nlp(text) \\\n",
    "                    if t.text not in STOP_WORDS]\n",
    "                # 这一步比较耗时，因此把tokenize的结果储存起来\n",
    "                data['tokens'] = tokens\n",
    "\n",
    "    # 根据分词结果建立词表，忽略部分低频词，\n",
    "    # 可以设置词最短长度和词表最大大小\n",
    "    def build_vocab(self, min_freq=3, min_len=2, max_size=None):\n",
    "        frequency = defaultdict(int)\n",
    "        for data in self.train_data:\n",
    "            tokens = data['tokens']\n",
    "            for token in tokens:\n",
    "                frequency[token] += 1 \n",
    "\n",
    "        print(f'unique tokens = {len(frequency)}, '+\\\n",
    "              f'total counts = {sum(frequency.values())}, '+\\\n",
    "              f'max freq = {max(frequency.values())}, '+\\\n",
    "              f'min freq = {min(frequency.values())}')    \n",
    "\n",
    "        self.token2id = {}\n",
    "        self.id2token = {}\n",
    "        total_count = 0\n",
    "        for token, freq in sorted(frequency.items(),\\\n",
    "            key=lambda x: -x[1]):\n",
    "            if max_size and len(self.token2id) >= max_size:\n",
    "                break\n",
    "            if freq > min_freq:\n",
    "                if (min_len is None) or (min_len and \\\n",
    "                    len(token) >= min_len):\n",
    "                    self.token2id[token] = len(self.token2id)\n",
    "                    self.id2token[len(self.id2token)] = token\n",
    "                    total_count += freq\n",
    "            else:\n",
    "                break\n",
    "        print(f'min_freq = {min_freq}, min_len = {min_len}, '+\\\n",
    "              f'max_size = {max_size}, '\n",
    "              f'remaining tokens = {len(self.token2id)}, '\n",
    "              f'in-vocab rate = {total_count / sum(frequency.values())}')\n",
    "\n",
    "    # 将分词后的结果转化为数字索引\n",
    "    def convert_tokens_to_ids(self):\n",
    "        for data_split in [self.train_data, self.test_data]:\n",
    "            for data in data_split:\n",
    "                data['token_ids'] = []\n",
    "                for token in data['tokens']:\n",
    "                    if token in self.token2id:\n",
    "                        data['token_ids'].append(self.token2id[token])\n",
    "\n",
    "        \n",
    "dataset = BooksDataset()\n",
    "dataset.tokenize()\n",
    "print(dataset.train_data[0]['tokens'])\n",
    "print(dataset.label2id)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "26d79e05",
   "metadata": {},
   "source": [
    "完成分词后，对出现次数超过3次的词元建立词表，并将分词后的文档转化为词元id的序列。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0d6b1918",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unique tokens = 6956, total counts = 54884, max freq = 1635, min freq = 1\n",
      "min_freq = 3, min_len = 2, max_size = None, remaining tokens = 1650,\n",
      " in-vocab rate = 0.7944209605713869\n",
      "[18, 26, 5, 0]\n"
     ]
    }
   ],
   "source": [
    "dataset.build_vocab(min_freq=3)\n",
    "dataset.convert_tokens_to_ids()\n",
    "print(dataset.train_data[0]['token_ids'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d096d95f",
   "metadata": {},
   "source": [
    "接下来将数据和标签准备成便于训练的矩阵格式。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ba632265",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "train_X, train_Y = [], []\n",
    "test_X, test_Y = [], []\n",
    "\n",
    "for data in dataset.train_data:\n",
    "    x = np.zeros(len(dataset.token2id), dtype=np.int32)\n",
    "    for token_id in data['token_ids']:\n",
    "        x[token_id] += 1\n",
    "    train_X.append(x)\n",
    "    train_Y.append(data['label'])\n",
    "for data in dataset.test_data:\n",
    "    x = np.zeros(len(dataset.token2id), dtype=np.int32)\n",
    "    for token_id in data['token_ids']:\n",
    "        x[token_id] += 1\n",
    "    test_X.append(x)\n",
    "    test_Y.append(data['label'])\n",
    "train_X, train_Y = np.array(train_X), np.array(train_Y)\n",
    "test_X, test_Y = np.array(test_X), np.array(test_Y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3938acdb",
   "metadata": {},
   "source": [
    "下面代码展示朴素贝叶斯的训练和预测。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f13251b7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P(计算机类) = 0.4453460067230787\n",
      "P(艺术传媒类) = 0.26660484525327466\n",
      "P(经管类) = 0.2880491480236467\n",
      "P(教程|计算机类) = 0.5726495726495726\n",
      "P(基础|计算机类) = 0.6503006012024048\n",
      "P(设计|计算机类) = 0.606694560669456\n",
      "test example-0, prediction = 0, label = 0\n",
      "test example-1, prediction = 0, label = 0\n",
      "test example-2, prediction = 1, label = 1\n",
      "test example-3, prediction = 1, label = 1\n",
      "test example-4, prediction = 1, label = 1\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "class NaiveBayes:\n",
    "    def __init__(self, num_classes, vocab_size):\n",
    "        self.num_classes = num_classes\n",
    "        self.vocab_size = vocab_size\n",
    "        self.prior = np.zeros(num_classes, dtype=np.float64)\n",
    "        self.likelihood = np.zeros((num_classes, vocab_size),\\\n",
    "            dtype=np.float64)\n",
    "        \n",
    "    def fit(self, X, Y):\n",
    "        # NaiveBayes的训练主要涉及先验概率和似然的估计，\n",
    "        # 这两者都可以通过计数简单获得\n",
    "        for x, y in zip(X, Y):\n",
    "            self.prior[y] += 1\n",
    "            for token_id in x:\n",
    "                self.likelihood[y, token_id] += 1\n",
    "                \n",
    "        self.prior /= self.prior.sum()\n",
    "        # laplace平滑\n",
    "        self.likelihood += 1\n",
    "        self.likelihood /= self.likelihood.sum(axis=0)\n",
    "        # 为了避免精度溢出，使用对数概率\n",
    "        self.prior = np.log(self.prior)\n",
    "        self.likelihood = np.log(self.likelihood)\n",
    "    \n",
    "    def predict(self, X):\n",
    "        # 算出各个类别的先验概率与似然的乘积，找出最大的作为分类结果\n",
    "        preds = []\n",
    "        for x in X:\n",
    "            p = np.zeros(self.num_classes, dtype=np.float64)\n",
    "            for i in range(self.num_classes):\n",
    "                p[i] += self.prior[i]\n",
    "                for token in x:\n",
    "                    p[i] += self.likelihood[i, token]\n",
    "            preds.append(np.argmax(p))\n",
    "        return preds\n",
    "\n",
    "nb = NaiveBayes(len(dataset.label2id), len(dataset.token2id))\n",
    "train_X, train_Y = [], []\n",
    "for data in dataset.train_data:\n",
    "    train_X.append(data['token_ids'])\n",
    "    train_Y.append(data['label'])\n",
    "nb.fit(train_X, train_Y)\n",
    "\n",
    "for i in range(3):\n",
    "    print(f'P({dataset.id2label[i]}) = {np.exp(nb.prior[i])}')\n",
    "for i in range(3):\n",
    "    print(f'P({dataset.id2token[i]}|{dataset.id2label[0]}) = '+\\\n",
    "          f'{np.exp(nb.likelihood[0, i])}')\n",
    "\n",
    "test_X, test_Y = [], []\n",
    "for data in dataset.test_data:\n",
    "    test_X.append(data['token_ids'])\n",
    "    test_Y.append(data['label'])\n",
    "    \n",
    "NB_preds = nb.predict(test_X)\n",
    "    \n",
    "for i, (p, y) in enumerate(zip(NB_preds, test_Y)):\n",
    "    if i >= 5:\n",
    "        break\n",
    "    print(f'test example-{i}, prediction = {p}, label = {y}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1cf6399",
   "metadata": {},
   "source": [
    "下面使用第3章介绍的TF-IDF方法得到文档的特征向量，并使用PyTorch实现逻辑斯谛回归模型的训练和预测。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "21a3bc79",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "\n",
    "sys.path.append('../code')\n",
    "from utils import TFIDF\n",
    "        \n",
    "tfidf = TFIDF(len(dataset.token2id))\n",
    "tfidf.fit(train_X)\n",
    "train_F = tfidf.transform(train_X)\n",
    "test_F = tfidf.transform(test_X)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dc8af30b",
   "metadata": {},
   "source": [
    "逻辑斯谛回归可以看作一个一层的神经网络模型，使用PyTorch实现可以方便地利用自动求导功能。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1ddebf0c",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "epoch-49, loss=0.2376: 100%|█| 50/50 [00:09<00:00,  5.36it/s\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAGwCAYAAABB4NqyAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAABZ70lEQVR4nO3deVhUdd8G8HsYVlkFZFNQ3FFcwQXcl3DPNNOy1EorszKzNE1LTd+0zbTcKjNbTK1cHssVS3HBFVFRcQdBBRFUVlnnvH8QwwyzMDPMytyf6+K65pzzO+d85zznab7+VpEgCAKIiIiIrIiNqQMgIiIiMjYmQERERGR1mAARERGR1WECRERERFaHCRARERFZHSZAREREZHWYABEREZHVsTV1AMYmkUhw9+5duLq6QiQSmTocIiIi0oAgCMjNzUVAQABsbGpef2N1CdDdu3cRGBho6jCIiIhIB6mpqWjQoEGNr2N1CZCrqyuA8gfo5uZm4miIiIhIEzk5OQgMDJT+jteU1SVAFc1ebm5uTICIiIgsjL66r7ATNBEREVkdJkBERERkdZgAERERkdVhAkRERERWhwkQERERWR0mQERERGR1mAARERGR1WECRERERFaHCRARERFZHSZAREREZHWYABEREZHVYQJEREREVocJkB7lFJbgwp1sU4dBRERE1WACpCcX7mSj/YJ9ePHHkxAEwdThEBERkRpMgPSkma8L7G1tkJlXjEtpOaYOh4iIiNRgAqQnDrZidGrkCQDYd/GeiaMhIiIidZgA6VHPZvUAANGX7rEZjIiIyIwxAdKjAa39AACX0nJwPSPPxNEQERGRKkyA9CjIqw5C/N0AAO/8fta0wRAREZFKTID0LMTfFQBw4Q47QhMREZkrJkB69kafptLPp5MfmDASIiIiUoUJkJ41qeci/TxqzTETRkJERESqMAEyMI4GIyIiMj9MgAzs4NX7pg6BiIiIqmACZACtA9yknyeuP2XCSIiIiEgZJkAGsGFSF+lniQDkFZWaMBoiIiKqigmQAXjUsZfb5qSIRERE5oUJkIGc/KCf9PNTK48yCSIiIjIjTIAMxMfNUW577eGbJoqEiIiIqmICZEB+MknQplOpJoyEiIiIZDEBMqDpTzQ3dQhERESkBBMgA7KxEZk6BCIiIlKCCRARERFZHSZABlSxMnyF2w8LTBQJERERyWICZECtA9wxOryBdLv7pwdw8W62CSMiIiIigAmQwX00rLXc9pa4OyaKhIiIiCowATIwZ3ux3LatmB2jiYiITI0JkIGJRCJ8PLyyFui7QzeRz7XBiIiITIoJkBGM69pQbnv+josmioSIiIgAJkBGIRLJN3v9EXcbgiCYKBoiIiJiAmQkR2f1ldu+8+ixiSIhIiIiJkBGUt/DSW67sERiokiIiIiICZCJZD8uNnUIREREVosJkIk8vfoYBi8/bOowiIiIrBITICOaECE/GuxSWg7m/e+CiaIhIiKyXkyAjKhrYy+FfT8du4UH+WwOIyIiMiYmQEY0oLWf0v1FpWVGjoSIiMi6mTQBOnToEIYNG4aAgACIRCJs37692nNiYmIQFhYGR0dHNG7cGGvWrDF8oHpiY6N8GYz+X8YYORIiIiLrZtIEKD8/H+3atcOKFSs0Kp+UlITBgwejR48eiI+PxwcffICpU6diy5YtBo7UsPKLy1BYUobiUg6NJyIiMgZbU9580KBBGDRokMbl16xZg6CgICxbtgwAEBISgtOnT+OLL77A008/rfScoqIiFBUVSbdzcnJqFLOh9Pr8AIpKJTg+ux8c7cTVn0BEREQ6s6g+QMeOHUNUVJTcvgEDBuD06dMoKSlRes7ixYvh7u4u/QsMDDRGqFq7l1OERwUlSLiTjbc2xuOHI0mmDomIiKjWsqgEKD09Hb6+vnL7fH19UVpaiszMTKXnzJ49G9nZ2dK/1NRUY4Sq0nOd1Sdg+y/dw1/n7mLh35eMFBEREZH1sagECFBcWLRiUdGq+ys4ODjAzc1N7s+UFo9sq/Z4blGpkSIhIiKyXhaVAPn5+SE9PV1uX0ZGBmxtbeHlpTjHjrmaNailymNcJJ6IiMjwLCoBioiIQHR0tNy+ffv2ITw8HHZ2diaKSnuTezVB2wbuKo4yAyIiIjI0kyZAeXl5OHv2LM6ePQugfJj72bNnkZKSAqC8/8748eOl5SdPnoxbt25h+vTpSExMxLp16/DDDz/gvffeM0X4NVImUZ7osAaIiIjI8Ew6DP706dPo06ePdHv69OkAgAkTJmD9+vVIS0uTJkMAEBwcjF27duGdd97BypUrERAQgK+//lrlEHhzpioBIiIiIsMTCYJ11Tnk5OTA3d0d2dnZJu0QHfVVDK7ey1NbJnnJECNFQ0REZN70/fttUX2AahNfN0dTh0BERGS1mACZyCcj2lRb5npGrhEiISIisj5MgEwk0LNOtWXe2XzOCJEQERFZHyZAZiynUPnyHkRERFQzTIDMmMS6+qcTEREZDRMgE4pson726tQHj6Wft8ffwbz/XYCEw+eJiIhqjAmQCX0/PhxdG3tqVHba5rP46dgt7L6QXn1hIiIiUosJkAk5O9hi3YudMLJDfY3PycovMmBERERE1oEJkInVsbfF0jHtkbR4sEbl2S2IiIio5ky6FAZVEolESvdfSc+VGw1mZRN3ExERGQQTIDPStbEnjt98ILdv0PJDkO33zPSHiIio5tgEZkZa+Loq7Ks66IsVQERERDXHBMiMvDegRbVlmP8QERHVHBMgM+LqaIcZ1SRBgiDg7qPHeFxcZqSoiIiIah8mQGbmhS4N1R6/cT8fkUv+RZ8vDhonICIiolqICZCZEVXzv8j+xHsAgPScQiNEQ0REVDsxATIztjbKh8NXUH+UiIiINMEEyMzUsVc/M0FGLmeCJiIiqikmQGbo0Iw+cHeyM3UYREREtRYTIDMU5FVH40VSiYiISHtMgMyUjYqlMYiIiKjmmACZqfoeTtWWeenHk7hwJ9sI0RAREdUuTIDM1Nv9m6GhVx21ZQ5cuY8x3x4zUkRERES1BxMgM+XqaIdvnutQbbl8zghNRESkNSZAZoz9gIiIiAyDCZAZ0zT/mfTTaQhcJp6IiEhjTIDMmESiWbn9ifcwe2uCYYMhIiKqRZgAmbESTTMgAJtOpRowEiIiotqFCZAZKynVPAEiIiIizTEBMmO2Yv7PQ0REZAj8hTVjHYM8MKJDfbQP9DB1KERERLUKEyAzJhKJ8NWY9vhgcIhG5TedTEFhCecFIiIiqg4TIAvQqVFdjcrN2pqA59eeQFEpkyAiIiJ1mABZAJFIhB7NvAEAofXd1JaNu/UQb288a4SoiIiILBcTIAvx9bMd8E7/5ljzQli1ZfdcTDdCRERERJaLCZCFqOtsj7f7N0ODuuoXSK3wxd4rBo6IiIjIcjEBskDLn21fbZkVB66jtIzzCBERESnDBMgCOdmJNSrXdM5urhFGRESkBBMgC6RNSpNXVGqwOIiIiCwVEyALpE2lTvdPD+CzPZcNFwwREZEFYgJkkTTPgLIfl2DVwRsGjIWIiMjyMAGyQLp062k0ayfiUx7qPxgiIiILxATIAunarXnEqli9xkFERGSpmABZIA7sIiIiqhkmQBaoc7AnAKBuHTsTR0JERGSZmABZoHquDoib2x/HZvczdShEREQWydbUAZBuvFwcTB0CERGRxWINkIX7+63uWPBka43LH7icgdPJD5CYloOvoq+ioJgTJRIRkfVhDZCFC63vjqY+Lpi346JG5V9af0puu6C4FHOGtDJEaERERGaLNUC1gKOdGJ+MaKPTueduZ+s5GiIiIvPHBKiWGNslCBO7B2t93t1Hjw0QDRERkXljAlSLRDbx0vqc2w8fY8e5uwaIhoiIyHwxAapFCkskOp23dN8VPUdCRERk3pgA1SKFJWWmDoGIiMgiMAGqRaJa+8LPzRFPtQ/Q6jyRSGSgiIiIiMwTh8HXIq6Odjg6qy/ENiJsP8t+PURERKqwBqiWEdtoX5tTXCpBSVn5354L6XiQX2yAyIiIiMwHEyDCnUeP0W3Jv1h98AYm/xqHUatjTR0SERGRQTEBIgBARm4RNp9KBQDczMw3cTRERESGxQSIpNj0RURE1oIJUC3Vs3k9AICdWPM+QY85jJ6IiKyEyROgVatWITg4GI6OjggLC8Phw4fVlt+wYQPatWuHOnXqwN/fHy+99BKysrKMFK3l+H58GHZN7YFnOwWZOhQiIiKzY9IEaPPmzZg2bRrmzJmD+Ph49OjRA4MGDUJKSorS8keOHMH48eMxceJEXLx4EX/88QdOnTqFSZMmGTly8+dgK0arADedlscgIiKq7UyaAC1duhQTJ07EpEmTEBISgmXLliEwMBCrV69WWv748eNo1KgRpk6diuDgYHTv3h2vvfYaTp8+beTILcfAUD/MHRKi9XnnUh/pPxgiIiIzYbIEqLi4GHFxcYiKipLbHxUVhdhY5cOwIyMjcfv2bezatQuCIODevXv4888/MWTIEJX3KSoqQk5OjtyfNRGJROjdwkfr84avPGqAaIiIiMyDyRKgzMxMlJWVwdfXV26/r68v0tPTlZ4TGRmJDRs2YMyYMbC3t4efnx88PDzwzTffqLzP4sWL4e7uLv0LDAzU6/ewBA62Ju/qRUREZFZM/stYdR0qQRBUrk116dIlTJ06FR999BHi4uKwZ88eJCUlYfLkySqvP3v2bGRnZ0v/UlNT9Rq/JQj0rKPTeflFpQDKF1ktKStfaf7ItUzEXs/UW2xERESmYLK1wLy9vSEWixVqezIyMhRqhSosXrwY3bp1w4wZMwAAbdu2hbOzM3r06IFFixbB399f4RwHBwc4ODjo/wtYgaHfHMHut3ug9by98HV1wL7pvfDCDycAAJcXDoSjndjEERIREenGZDVA9vb2CAsLQ3R0tNz+6OhoREZGKj2noKAANjbyIYvF5T/CgiAYJlArlpSZjxl/nkeZRMDd7EI8lJko8eLdHBRy3iAiIrJQJm0Cmz59OtauXYt169YhMTER77zzDlJSUqRNWrNnz8b48eOl5YcNG4atW7di9erVuHnzJo4ePYqpU6eic+fOCAgIMNXXqNX+Ole5qnzFUhkA8PTqWIxcxTXDiIjIMpmsCQwAxowZg6ysLHz88cdIS0tDaGgodu3ahYYNGwIA0tLS5OYEevHFF5Gbm4sVK1bg3XffhYeHB/r27YtPP/3UVF/BqmTmFcltX0qzrhF1RERUe4gEK2s7ysnJgbu7O7Kzs+Hm5mbqcIym0aydctst/VxxOT1Xq2uMDm+A30/fltuXvET1FARERET6ou/fb5OPAiPju/HJYEzr31zr885yckQiIqolmABZiYrZoBc9FQqxjQgSHSr+rt7L03dYREREJmHSPkBkPJN6NMYzYYFwr2MHACiTWFXLJxERkRzWAFmRiuQHYAJERETWjQmQlfJx4+SQRERkvZgAWamIxl5wdWQLKBERWScmQFZKJBLhpW7Bpg6DiIjIJJgAWbHxEQ3h6Wyvl2vdffQYL68/hSPXuFAqERGZPyZAVszbxQGn5vTHOzrMCVTVB9sS8O/lDOliqUREROaMCZCVE9uIUKaHycAzcoqqL0RERGQmmAARJHoYEm9vy1eJiIgsB3+1CKX6SIDEfJWIiMhy8FeLMKydPwCgqY+L1udm5hXhr3N3cTL5gb7DIiIiMhhOBENoHeCO2Fl94eVijxZz92h17gtrT2i9qjwREZGpsQaIAAABHk5wsBXjm+c6oEczb43PY/JDRESWiAkQyRnWLgC/TOxSo2ukZxci4Xa2xuXLJAJ+OJKEC3c0P4eIiKgmmACR3nVd/A+GrTiC6xmVtUMlZRKsPXwTl9NzFMpvOpWChX9fwtBvjhgzTCIismJMgMhgzsvUAq0/moxFOxMxcNlhhXKX7iomRURERIbEBIgMRmwjkn4+cCXDhJEQERHJYwJEGhncxk/rc67dywMAHLp6H7E3svQdEhERkc6YAJFSLXxd5bb7h/hqfY0VB64DAL7+55racjWfhpGIiEg7TIBIqe1vdMOW1yOl2yKRmsLVsBXX4GQiIiIDYAJESjnZi9HCr7IWyEbHDCgzrwj3c7lQKhERmRfOBE0aEemYAIUv2q/nSIiIiGqONUCkkiBU9s4xZCOWwE5ARERkZEyASCVbm8rXo0uwJ1wdbBU6RxMREVkiNoGRSk72Yix/tj1KywT4uDni9If9YWtjgyYf7DJ1aERERDXCBIjUGt6+vvSzg63YQHepbAOLvZGJyCaaL8ZKRESkCzaBkdb+fqt7jc6XSMoTnkt3czDpp1NyK8qP/f4E7j56XKPrExERVYcJEGkttL47Wvrp3hdo3o6LAIDBXx/G/sQMxKc8kjvOBIiIiAyNTWBkdL8cv4WODT00Li+RCLCx4WSKRESkP6wBIp0sebptjc5/Z/M5jcr9e/ke2i3Yhz0X0mt0PyIiIllMgEgnbeu7G+zasnMuvrz+NHKLSjH51ziD3Y+IiKwPEyDSiUikuGAqERGRpWACRDoRiUTY9XYPvNmnqalDISIi0hoTINKZ2EaEd6Oa6/26V9Lz5JbhICIi0jcmQFQjui6Sqs4H2xLQ6qO9uJ6Rp/drExERAUyASA/eH9hS79d8XFKGeTsu6P26REREABMg0oPXezfB5YUD9X5diUR++8KdbJxJeaj3+xARkfXhRIikF452+l8nTIB8P6Ch3xwBAJybFwV3Jzu934+IiKwHa4DIbKnqB/0wv1hhX2JaDpbtv4qC4lIDR0VERLUBa4DIbJ1IeqB0v42SjteDlh8GABQUl+GDwSEGjYuIiCwfa4BIb6Ja+QIAlj/b3qD3UTfw7MKdbIPem4iIagedEqCffvoJO3fulG7PnDkTHh4eiIyMxK1bt/QWHFmWNS+EIf7DJzC8fX2M7FjfYPdRtzCqAUblExFRLaRTAvTJJ5/AyckJAHDs2DGsWLECn332Gby9vfHOO+/oNUCyHDY2ItR1tgcALB3dHslLhsDXzUHv92GOQ0RENaVTH6DU1FQ0bVq+BML27dsxatQovPrqq+jWrRt69+6tz/iIFJxLfQQHWxt8/c81jO4UiNYBlQuzipgeERGRBnSqAXJxcUFWVhYAYN++fejfvz8AwNHREY8fP9ZfdERKvL7hDMIW7cdPx25hyNdHFI4LgoDrGXkoKZMoOZuIiEjHBOiJJ57ApEmTMGnSJFy9ehVDhgwBAFy8eBGNGjXSZ3xk4Ya0CQBgvJXjRSLgf2fvov/SGLzy82mj3JOIiCyPTgnQypUrERERgfv372PLli3w8vICAMTFxeG5557Ta4Bk2WYObIHlz7bHxle7Gu2e644mAQAOXrlvtHsSEZFl0akPkIeHB1asWKGwf8GCBTUOiGoXRzsxhrcvHxE2d0gIFu1MNHFEREREOtYA7dmzB0eOVPa9WLlyJdq3b4+xY8fi4UOu1UTKTerRGL9N6mLQexy+lonztzkXEBERqadTAjRjxgzk5OQAABISEvDuu+9i8ODBuHnzJqZPn67XAKl2iWzqbeoQiIiIdGsCS0pKQqtWrQAAW7ZswdChQ/HJJ5/gzJkzGDx4sF4DJCIiItI3nWqA7O3tUVBQAADYv38/oqKiAACenp7SmiGi6vi46meSxMKSMpXHMvOKVB77+K9LeP3XOAiqVl0lIqJaS6cEqHv37pg+fToWLlyIkydPSofBX716FQ0aNNBrgFR7+bk76uU63T/9V+Wx8EX7sedCmtJj644mYfeFdFxKY9JORGRtdEqAVqxYAVtbW/z5559YvXo16tcvH+Wze/duDBw4UK8BUu1VteLFVs0aX+pk5hWrPb40+qra42US1gAREVkbnfoABQUF4e+//1bY/9VXX9U4ILIeAgR0alQXp5LLRw4aaiHTq/fy0GjWTvz4Yif0aeljmJsQEZFF0SkBAoCysjJs374diYmJEIlECAkJwfDhwyEWi/UZH9ViggDUsdf5FdTaS+tPIXnJkP/uXVnrU1LGGiAiImujUxPY9evXERISgvHjx2Pr1q34888/MW7cOLRu3Ro3btzQd4xUSwkCsODJ1vBzc8SHQ1sZdSFT2ea3p1fHIvZ6ptHuTUREpqdTAjR16lQ0adIEqampOHPmDOLj45GSkoLg4GBMnTpV3zFSLdbI2xnHZvfFxO7B8KhjZ7T7Sqp0QJq1NUFt+Rv387Am5gYeF6secUZERJZDp/aHmJgYHD9+HJ6entJ9Xl5eWLJkCbp166a34Kh2q0hBRP91/jFUHyCF+woCRq6OldtX3b37fRkDAHiYX4zZg0MMFRoRERmJTjVADg4OyM3NVdifl5cHe3v7GgdF1qG5r4vctrGawB7kF+u8XMaZFC71QkRUG+iUAA0dOhSvvvoqTpw4AUEQIAgCjh8/jsmTJ+PJJ5/U6lqrVq1CcHAwHB0dERYWhsOHD6stX1RUhDlz5qBhw4ZwcHBAkyZNsG7dOl2+BpnIX292x/iIhpg/rLXcfmPVAIlqcCPOmUhEVDvo1AT29ddfY8KECYiIiICdXXm/jZKSEgwfPhzLli3T+DqbN2/GtGnTsGrVKnTr1g3ffvstBg0ahEuXLiEoKEjpOaNHj8a9e/fwww8/oGnTpsjIyEBpaakuX4NMpE0Dd7Rp4K6w3xj5z7b42+jVXHEovLp7x7PWh4io1tEpAfLw8MD//vc/XL9+HYmJiRAEAa1atULTpk21us7SpUsxceJETJo0CQCwbNky7N27F6tXr8bixYsVyu/ZswcxMTG4efOmtP9Ro0aNdPkKZIaU1czIzhOkD+9sPodTc/prdG8ASM7Mx4hVsUqPERGR5dI4AapulfeDBw9KPy9durTa6xUXFyMuLg6zZs2S2x8VFYXYWOU/ODt27EB4eDg+++wz/PLLL3B2dsaTTz6JhQsXwsnJSek5RUVFKCqqXA+Ka5WZr0k9grHgr0to18Ad5/7rozNvWGsM/eaIXu8T9VWM0v3fH7oJfw9HDG0bIN2XyGUyiIhqJY0ToPj4eI3Kadq/IjMzE2VlZfD19ZXb7+vri/T0dKXn3Lx5E0eOHIGjoyO2bduGzMxMTJkyBQ8ePFDZD2jx4sVYsGCBRjGRab0Y2Qidgz1Rz8UBnT/5BwDgaKf/iTUfFpQo7EvKzMf/7UoEALkEqOrrzC5ARES1g8YJ0IEDBwwSQNWESRAElUmURCKBSCTChg0b4O5e3odk6dKlGDVqFFauXKm0Fmj27NlytVc5OTkIDAzU4zcgfRGJRGgd4I6H+ZVre7k62qJrY08cv/nAhJFV4srxRES1g06jwPTB29sbYrFYobYnIyNDoVaogr+/P+rXry9NfgAgJCQEgiDg9u3bSs9xcHCAm5ub3B+ZtzoOlbU+HnXssPGVrka9/7nUR+j9+QHsu6i8JpKIiCyfyRIge3t7hIWFITo6Wm5/dHQ0IiMjlZ7TrVs33L17F3l5edJ9V69ehY2NDRo0aGDQeMl4HGzFiJnRG4dm9IGDrRgikQhLR7cz2v0n/nQKyVkFePWXOKPdk4iIjMtkCRBQ3rF67dq1WLduHRITE/HOO+8gJSUFkydPBlDefDV+/Hhp+bFjx8LLywsvvfQSLl26hEOHDmHGjBl4+eWXVXaCJsvU0MsZQV51pNsjOzYw2lIZ+UWVy10cvZ5llHsSEZFxGW8pbiXGjBmDrKwsfPzxx0hLS0NoaCh27dqFhg0bAgDS0tKQkpIiLe/i4oLo6Gi89dZbCA8Ph5eXF0aPHo1FixaZ6iuQEUU09sLuC4ZvliqT6efzy/FbcsfYA4iIqHYQCVbWqzMnJwfu7u7Izs5mfyAL8zC/GL8ev4W72YXYeDKl+hN0ZGsjQqlE+f8tOgR5YNsUrndHRGRs+v79NmkTGJE26jrb461+zdCgrmGbO6uuFE9ERLUPEyCiKlRU/hARUS3CBIgsjo2xVk1VoqJy6OdjydhzIc1kcRARUc2YtBM0kS5MmP8AAK5n5OGj/10EACQvGWLaYIiISCesASKLNq1/M5z7KAr1XB2Mds8HMjNV6yK3sAS3svL1FA0REemCNUBkcWxkaoCm9W8OADBWpdDZ1Ec1Xg4jcvG/yC0qRfQ7PdHM11VPkRERkTZYA0QWx91JcUJEO7HxXuUx3x2Xfn5cXIZByw9j0d+XUFwqwc7zacjKK1I4535uEbaeuY3CkjLkFpUCAGKu3jdazEREJI81QGRxRnRogINX7qNbU2/pvpXPd8RTK48aPZYtZ24jMS0HiWk5qONgi6//uYZGXnVwcEYfuXKj1sTiVlYBEtNyjB4jEREpYg0QWRx7WxusfiEML3RtKN3XPtAD5z6KMnosc7dfkH7enVA+Kiw5qwC3HxagsKQMh6/dR1FpGW5lFQAA9l26Jy0vMnVvbiIiK8YaIKo13I20Vpgqdx89ln7u/ukBNKnnjBv389G1sad0vymH8BMRUSXWABHpSX5xmdz2jfvlI72O33wg3Seb/jAVIiIyHSZAVCvZmGl2kfdfB2giIjItJkBERpSRWzlCjK1hRESmwwSIapUuweX9bZ7u2MDEkRARkTljJ2iqVb4bH47D1+6jX0tf/BF329ThqKWqAkgQBJRJBNgacW4jIiJrw//CUq3i7mSHoW0D4GQvrrasl7O9ESLSXHGpBBfuZOPVX+LQ+ZN/2F+IiMiAmABRrfXJiDamDkGtRTsTAQAFxaX4bM9l9PniIIZ+cwTRl+7hQX4x9svMGURERPrFBIhqrSFt/RX2Te3XDAAwOtz0fYRKJeVrirX6aC9WHbyBOzLzCMmSSGq29hgRESliAkS1lq2SsfDT+jXD3291xycj2pjFKKzCkjK1x1ceuI62C/bh6r1cI0VERGQdmABRrSVWkgDZ2IgQWt9doYPx769FGCssOeoWlhcg4PO9V5BXVIoX153E3O0JSMtWXktERETaYQJEtZbsCvHPhDXAb5O6qCzbOdhTacJkaInpmi2Oeje7EL8eT8GUDWcMHBERkXVgAkS1lthGhB9f6oQ1L3TE58+0Q6TM6vHKNPSsI7cd/U5PQ4YHABi5Klar8hfvlK88/+rPp3ElvfpmsZIyCRb9fQkxV+/rGiIRUa3EBIhqtT4tfDAwVLEzdDn5Gp/vxodJPwe4O6KZr6sBI9Pd6DXHsO/SPTz73bFqy248mYK1R5IwYd1JI0RGRGQ5mAAR/aepT2XCo8k8QoZ2Kvmh0v25/80P9LCgpNpr3H7IPkNERMowASJSQvTfELEXugaZLIbfTqTU+BqCul7WRERWjAkQkRrzhrXGt+PCqi9oppj/EBEpx7XAyGqpmweo4pCd2AYDWvsZJR6NKIl5/6V7+N+5u/B3d8S4rg0RWKUzNxERKWICRKSEOUySqExxqURh36SfT0s/7zyfhqOz+hozJCIii8QmMCIlfFwdTR2CTlQtp1FTEomAx8XqZ60mIrIkTICIZKx/qRO6N/XGkqfNeyFVTSzbfxW/n07Vy7VGrYlFyEd7kJVXpJfrERGZGpvAiGT0buGD3i18TB1GjV28m41l+6/p7XpnUh4BAP65nIHR4YF6uy4RkamwBoislpl286mxguJSPNJgjiAiImvGBIhIA/97oxue6xyEuUNCTB1KtVp9tBf/JGaYOgwiIrPGJjAiDbQL9EC7QA8AQB17W3ywLcG0Af3nropOz+uOJhk5EiIiy8IaICItje0ShCuLBpo6DADA67/GaVxWEAR8sC0BS3ZfNmBERESWgQkQWa3mNVjs1MFWjNNz++sxGt2cv5Otcdm4Ww/x24kUrIm5YcCIlFsTcwNPr45F/n/rmBERmRoTILJaX45uh2fCGuCvN7vrdL63iwN2TtXtXH3RZqmLrPximfOqP1EQyuf+Sc8u1CqmuFsPsOmk/DpmS3ZfRtyth/j1+C2trkVEZCjsA0RWy9fNEZ8/065G12gd4C633bN5PRy6er9G1zQU2ZxHEKqf7frNjfHYeT5Nbp8mI+eeXn0MABDkWQeRTb3ljhWWKM5kTURkCqwBItIjV0fl/6bo0cxb6X7jqsyAlv1zDdnVDJWvmvxUuJ9bhD0X0lFapj6ZScrK1z5EIiIjYQJEpEdzBofAxUE+Cera2BO/TOxioogqSWRqgL7+5xrafbwPl9NzlJbNLVSeHEkEAYO/PozJv8ZhfWyy2vspa2UTUPPl6Y9cy8R7f5xDjooYiYg0wQSISI8CPJxwYcEAuVFiFYlAIy/TrtK+6uB1hX0vrD0h/Rxz9T6eX3scqQ8KsC3+jtJrvL8lAfdzy5fD2J94T+sYtOmzpMoLP5zAn3G3sXTf1ZpfjIisFhMgohqaNaglAGDGgBbSfQ62YoVyv04ybS3QhTuKtT2ZeZUdoyesO4mj17Mw88/zxgxLZ6kPCkwdAhFZMHaCJqqhyb2aYGSH+vBxU76CfEWlR4O6pq0BUudeTuVIr2M3szCojZ/W18gpLMH8HRfVltFDBZBBrkVE1oc1QER6oCr5sRTL9ss3J21X0QSmzlfRV7H1jPbn6UqTofxERKowASIyNCW/0/U9nIwfhxpVm8cqVn9Xp2r+oWpZDrUn1YCE+Q8R1QATICIDUzbyyclesY+QqXy25zIStJhRusKJpAfIfqzdSCw2gRGRuWACRGQgr/ZsDAB4f2BL6b52DconThzZsT46BnmYIiwFqw7qvjTGygPlI8sSbmfjSnqu3DFDJyiazmb9/aGbOHzNPCenJCLTYSdoIgP5YHAIpj/RHI52lbU9v0zqgrMpjxDZxAtjOweh/cfRJoyw5n4+loyp/Zph2IojGpU3dredw9cy8X+7EgEAyUuGGPfmRGTWWANEZECyyQ8AuDnaoWfzerAV28Cjjr1C+bFdgowVml4UlkjwUGaNseroYyJE6bU0uNQdmX5JuxLSkJatQT8lIrIKrAEiMhP/vNsL9T2c8NuJlOoLW4AbGXkGvb62ydSUDWdgL7bB1f8bZKCIiMiSsAaIyAy09HNFk3oucLQT472o5qYORyuqFlVVtlSGPpvAJDqsq1pczfplRGQ9mAARUY2IqltWXgOFJWVYeeC6yrXJlNGkBqjmkRFRbcUEiMjMPNtZeT+g9oEexg3EAFSlLGtibuDzvVcwcNlhpcdPJj3A67/GyfXhUVWbxAkSiUgTTICIzIy3iwOuK+mnsv2NbiaIpmbKJALWHUmSbqvKTc7fVj8P0ehvj2H3hXS5dcrSsgsVyi3enYhenx9EdgFXiici9ZgAEZmBqs1ItmLL+b/mx3+pXv9r48kUfPz3JYX9j4vLdLrX7YeVNUApDwpwOvkBCksqr/VtzE2kPCjAL8eTdbo+EVkPjgIjohrZe/GeymPK1hRbGn0VX/9zDT+93Bm9mtcDoHmzVdXuRqPWHMPA1n4olQio51o5rYAurWBHr2diS9xtfDSsldIpCoiodmECREQGc/rWQ7ntNTGVs05PWHdSOjmhpvmKsk7Ney6mqy6vRS/o59eeAADYiW3w6ai2mp9IRBbJcurZiUipNvXLl9doXM/ZxJFoLz7lIXIKS3D4WqZ03/CVR+WatXRRk4FpspMnSiQC4lMeSpvsJBIBs7acx09KhvgTkWVhAkRkBpT9Xi8c3lrtOX1b+qCRVx389koXLH+2Pf54LcIwwRlQzNX7iFp6CGUyS7ufS32ErWfKm86KSyVyzWP6GHKvjV+O38KIVbGYsO4kAODI9UxsOpWKeTtU93siIsvAJjAiMxDo6aSwb1xEI/x7OQMHrtyHr5uD3LGOQR74YUI4JAIgthFhePv6xgpVr5btv6Z0/+nkBxjazh/dFv+L9jKLxmqa/tQkUTpyPRODlh/GZ0+3lc7KfTL5AQRBQG5hqc7XJSLzwgSIyIQ2vdoVvx6/hY+GtVJ6/OvnOmDzqVQMbuMvt//psAYQiUQQV/mdb1PfHQl31A8ptwRb4+/g/J1s5BaVyjWPXdPD8hrXM3LR1MdVbZnEtBxM+PEk6rlUJp4Hr3JFeaLahE1gRCbUtbEXVoztCB9XR6XHXR3tMKlHYwR4KNYQKTMuoqE+wzOp6zVMdlKyClBSpti9uv/SQxqd/6igWK4vkewQfCKyfCZPgFatWoXg4GA4OjoiLCwMhw8rnwm2qqNHj8LW1hbt27c3bIBEZkjVMO+nOzbA672boL6HE2yseB2I4zez0PPzA5i7/YLS41M3xkMiUT/2rOphK36cRLWSSROgzZs3Y9q0aZgzZw7i4+PRo0cPDBo0CCkp6lfDzs7Oxvjx49GvXz8jRUpkGcQ2Irw/sCWOzuqLywutd9Vz2WYzZXacu4sTSQ+0uqZIpP0K9JbkSnou1h9NQikXjCUrYdIEaOnSpZg4cSImTZqEkJAQLFu2DIGBgVi9erXa81577TWMHTsWERGWN+qFSB80+Rm2tzV5Ba9Zy35cXG0ZY486M6UByw5h/l+XsPGk+n+AEtUWJvsvZHFxMeLi4hAVFSW3PyoqCrGxsSrP+/HHH3Hjxg3MmzdPo/sUFRUhJydH7o+IqEDL5ThEKhrByiQCkjPz9RGSWagNneiJNGGyBCgzMxNlZWXw9fWV2+/r64v0dOUzu167dg2zZs3Chg0bYGur2QC2xYsXw93dXfoXGBhY49iJyPKVVdMHCACy8oqkn0Ui+SRo/o6LyCkswbTNZ9H7i4P4/VSqQeI0Nl2WESGyRCavI69axSwIgtJq57KyMowdOxYLFixA8+bNNb7+7NmzkZ2dLf1LTa0d/5Ei0sQXz7ST227bwB3XlKw0b40q/jtz9V4u4m4p7w+UkVukdD8ArI9Nxqe7L+Ovc3cBADO3nFdZlojMj8nmAfL29oZYLFao7cnIyFCoFQKA3NxcnD59GvHx8XjzzTcBABJJ+Syxtra22LdvH/r27atwnoODAxwcHBT2E1mDUWENMLStP1p+uAdAea2HndgGK8d2xBu/nTFxdKYlAlBUWoaorzQbFq+sAUwf8xIpIwgCHhaUwNNZ90VZb97Pw7WMPAxo7afHyIhqD5PVANnb2yMsLAzR0dFy+6OjoxEZGalQ3s3NDQkJCTh79qz0b/LkyWjRogXOnj2LLl26GCt0ItPTop3C0U6M3i3KV11/qVswAGBwGz+81K0RBrfxQ7+WPnoJaWQHy5qN+o+4VLSYu0fj8ieTH2DDiVty+wzVRXrmn+fRcWE0DlzJ0Pkafb+MwWu/xOFINSPiiKyVSWeCnj59OsaNG4fw8HBERETgu+++Q0pKCiZPngygvPnqzp07+Pnnn2FjY4PQ0FC58318fODo6Kiwn6i207abxvfjw5GcmY+mPi4Aypt/5g2rXGus0aydNY7ps1FtsTX+To2vYyzHb2o3DL5ifTJj+CPuNgDg63+uoU+LmiWo524/Qvdm3hqXZxcgshYmTYDGjBmDrKwsfPzxx0hLS0NoaCh27dqFhg3LZ7NNS0urdk4gIqqendgGzXxVL//gbC9GvpajoqqyFZu8S6HRVZ1LSFUfRl3FpzzC3+fvYmjbAL1dk4jKmfy/WFOmTEFycjKKiooQFxeHnj17So+tX78eBw8eVHnu/PnzcfbsWcMHSWRmKmpy9CXuwyfwQtcgnc/f907P6gtZgT0XlI9grYk3f4uvtszf5+/ipR9P4lFB9XMbEVE5kydARKS5v9/qjq/GtENkE82bNDThaCeGi4Odzuc3V1O7ZE02nkqVW8OstEyCU8kPUFhShrhbD3E53TDzkL35WzwOXLmPr6KvKhzTtkKKw+DJWjABIrIgofXdMaJDA4Nc24omPTaYQ1fvo//SGFy9lwsA+Obf63hmzTGMX3cST6+OxcBlmq11qEqZRMDMP8+pnK35p2O3sOW//kNEpB4TICJS8NsrXXBl0UCcnttfbnRXSz9XzB0SYsLILEPFyKsfjyYBAE5que6YKtGX0vH76duYvTVBZZl3/ziH7IIS6XZZmYDCEt37d128m42fYpOrXTyWyNIwASIiBZFNvOFgK4a3iwNGhVXWOO2Z1hOTejTWyz2WjWmvl+uYo4//voSSMglcHPQ7ziTncan0c+yNTKRkFSgt91gm4fky+iraLdincxI05OsjmLfjIracYc0S1S5MgIhIrYgmXhjXtSEWDm9dfWEt1HR4t7k7cDkDd7MLVR5PfVCAG/d1n0hx7Pcn0PPzAxqVLSqVaHyvLWduK21iS0zL1So+XQmCAIEdkcgITDoMnojMn0gkwsKnVM+19ftrETpeWMeALMSha/eV7j+T8hCX03LxwbbyZqzlz7bHqgM38MUz7RBa3039MHolhy7d1axjtTY5xeytCXius/yoQGP0EZNIBAxfeRRuTrb4dWIXvU4pQFQVEyAiAqB7PtI52FP6ed87PTVeWsKmlv+2/XpceUflJbsu42RyZZ+gtzedBQCM+e4YPJ3tpbN2y3r/z/PYfDoV3Zp6KRwbtSZWPwFXEZ/yEO0DPaTbxvif62ZmnnQ1+jKJAFtxLX9JyKTYBEZEeuPr6ij9/MHglmrL8l/38gqKy3D74WOlidPm0+WLOB+9nqX0PE1o26o0YlUsRqwyTHKlCb4fZGhMgIhIb0Qy/0UZ1k797MXW2s9DtvbHUGJv6Gf9r7Opj6SfK/IRQRCqHRFWpocRY3lFpdUXIqoBJkBEpLVfJnZG3Tp2WPNCmNx+scy/2iUCEOztrNV13RzZKq8Pm06lKuwTqqzyFXs9Ewd1WGz15fWn0PfLgygqVV7zlFNYgi6f7Mfbm6qfwVqdd38/W6PziarDBIiIAGjXybVHs3o48+ETGBjqJ7ffRjYBkgjYNiUS61/qpPQadkrWDpsxUH2zGWlG2bxDspUyJWUSjF17Ai/+eEpuziB1KpqkDly5j+SsAsTdeqi03F/n7iIzrxj/O3tX+8Bl7E/UPjkj0gb/uUVEOlHWR8NGJqcR24jgUccevasMd//imXawtRHB0U6scH5YUF29x0nlZJscS8ok0s85hSVwr1P9Miia5Mf6aPoiMhYmQESkNw62YrzWszHyikoR4OEk3e/uZIfsxyVo6ecqN7GirM7BnmgV4GasUK1OVp5hF0pNzszH0G+OwMFWdcPCn3G3cTktBzMGtoCDrWICrIm8olK9TjCZ/bgEtjYiOOt50koyf2wCIyIAgL1Ytx+kqmYPDsH/jWgjt2/L65F4rnMgvh8frvK8rv8Np9f3SvdUbtLPp5Xu17gvejVVQF9GX0VeUSmy8pUnWncfPcZ7f5zD2iNJ+PivSxreVN7vp1MROm+vdImRqqIv3cNH/7sgV8OlzuPiMrRbsA+t5+3VKR6ybEyAiAgA8FL3Rmgd4IYZA1ro/dpNfVyweGRbBHrWUVlm4n9LbOyd1lNubiHSn8NKJmfMLy7F4t2JWl9r7Pcn8EAm2amuiUy2BmrDifKh/kv3XcEH2xJkmufUX2Xmn+cBAAv+S6AOXb2PYzcqpwZ45efT+PnYLZWLxVZ160G+RuWodmICREQAADdHO+yc2gNv9Glq9Ht/NLQV3J3K+6GIbURqm1EOz+yDr5/rYKzQapVxP5wEIN8hemn0VXwbc7Pac0VKkpOvoq+isKQMi/6+hBNJinMUyZ0vUvz89b/X8duJFFzL0H5JkOyCEoxfdxLPfX8cpVVqfNLULEEiF5PMd7LWaRmsGRMgIrIYe6f1RKBnHfRrWbvXETO0rLwi6WdNl9IQiRSThLyiUnx/6CbWHknCvZwipedl5BRizLfH8Ne5ylFhdjbyPz1RXx3Coav3gSpD9WXPqSozv/J+pVU6X+uSyzD/sT5MgIjI7FT9MWrp54ozHz6BFn6uAMpriUg3f5+/i16fH5Ru33n0WKPzBAGIl5kYEQAeFhTjy+irKs8pLZNg8e7LOJH0AN8eqqxlKi6TKPTTGb/upML5b22snEvofq58gtXvyxiV910TcwP5Wk6k+P6W8/hi7xWtzrEEZRIBm06m4LoOtWy1HRMgIjJ7617sBE9ne+m2uiYyUu/N33SboHDD8VsYWWVpjINXlC/4WmHjqVRk5imvGfr6n2ta3f/9Lee1Kr/iwHWtyv8Rd1vrcwwtI6cQhSWaLXWiyuZTqZi1NQH9l6pOGK0V/ytCRCZXdUoh2VmLY2f1lRtSX15ehCda+crtGxTqhxVj2TfIUHJ1WJpiWfRVHL6mfFmO308rzlatzNrD5TVHiWmaNdVVuKukZuvLfVewScMO0vp06W4OVh64juJSzUanAUBKVgE6f/KP2pouTZxJUT5hJTEBIiIzULVBy1amj0jV5EfZOW/3a4bVL4RhaNsAbH+jG1oHuOGnlzvrP1DSiqoh8QBU9hmqatHO8hFqNmqmKpdo0IFnTcwNfPPvdczamqC2nD46Qy/8+xL6fHEQuYXls2wP/vowPt97BWtibmh8jYNXy2fC1rSJUhVj921StUSKOWICRERmZ/6TrVHfwwkLh7fWqPw7TzSXfm4f6IGdU3ugWxMvQ4VHBnJERW0RoH6pFk1+5JfsvqxRDMqulf24BL+fSkX2Y82WDfnhSBKSMvOx6aR8LZfsIrWFJWUKo9dk6auZt+oacIb09/m7aDF3j8bTEJgaEyAiMjvB3s44OqsvxkU0Ulmmuv+sq6sx0MTHGiZfpD/zVUyQ+L+zd2pcA6SMsksqu9JbG+Mxc8t5vP+ndv2QqtaGPC4u3y4sKUOb+XvR58uDKs/VdaZsU6roXza7mlo2c8EEiIgsUnW/ecp+3Ho2r6fx9Ye08dcyIjKUtzedxcMC1c1p2qY/hSVleGrlUXy5T3HUV0UytfFkCnacuwuJRPhviD6w52K6Qvk/425j7PfH8UhJfAp9fv57KRPTclBSJiD1germLdnFgjVplssvKsWamBtIzqwyuSOH96vEBIiIaiWRSIT6Hk6oY1/5L+mqHadV6dzIE14uDoYKjXSQW6i6E/adh4qJxKGr99Fo1k6M/vaYwrFt8XdwNvUR9l68p3BMIgi4cT8Ps7cmYOrGeLnzHWxtUFhShuhL95D3X6fw9/44h9gbWfjmX8URZEVVmrgqcnJNchLZBF6TNWaX7L6MJbsv44mvjDva63pGHt774xxuZVnerNpc/Y2ITK6OgRaiPDijN8okAlp+uAcAINHgl+Ta/w2CrZJ5hlwdbdX+CJPpDFp+GMlLhsjte1hQ3l/nZNIDfLBNvklG3WgsQZBftuP0rcpRVCIR8MmuRPx87Ba6NfXChkldpcdyC0vw24kUeLvYy11LljatsrJFJYIAcTXLhFTMxF1SVmVSyGruIwgC8opK4epop3lwMkaticWjghLE3bK80WasASIik/lwaCv0D/HBU+3r63B29cmMndgGjnZiPN2xAbxdHBTu89ukLkrPESn5peJMwZbrtxPynXLV9RkSBNXHRRBJO/gevS6/9EdyVgE+2JaAV3+Jkylf9XzNydcACbiXU4gDlzO0HqUmW37WlvOYu10+GZz++zm0mb8P56pMcqmpR/8lmklVm94sAGuAiMhkJnYPxsTuwTqdq83vwJej26FMIsjNIN3K3w2RTb3x77u90FeDuVZ07WhLxrE9/o7GZReoWY1egIBZaiZdlK1ElJ3NOkfJCLEyiaCXYfWCAPT49ACKyyRY/mx7ZD8uQdsGHmgf6KHVdTadKh+V1ru5D34/nYoFw1tj23/P7dtDN7Dq+bAax2pJmAARkUV6opUv/rmcIdfkoE7V5TMq/oXduJ6LRufrOwHaM60HBi47rNdrWrNpm8/q5ToSobw2RxXZhEZ2kkLZTssV1h5JQsKdbOm2spFsgiAorXGUrS8qkwgo/i/ZmrPtgrT/UdVmP6XxKtk36efTAIAimaZAZYvd1nZMgIjIIo0OD4SvuyPa1nc3yv0kmk/iqxFLHOZsDcrU9BOrmqekPKhMlOzEyhOIE0kPFM7/VmZCREFQ3jeoahNYhTwdZuRWRW62bBX5z+qDNyC2AV7t2URv9zUX7ANERBbJxkaEPi18dB6tpU2HVE9ne3Rp7Kmwv0uw4j5NKetoTabXbsE+lcdEUN3zzF6DiQsrallkR5+tOaR8dmj5TtDKr5eWXbNZomX/P6DsbXyYX4xP91zGJ7suo6C49g0AYA0QEVE1vF3s8dWY9vgpNhmjwwOx4UQKvJztMT6yIVrM3aPTNVUt8UHmTVVLaNUmVmVOJj9QmE36sz1XUM/FAetjk7H82Q5o6qPYJKtq9GLE4n+x/qVOuHovV2kn5Pk7LuJ/Z++qjEe22Uu2ee7CnWz8fT4No8IqBw2UajIW38IwASIiqoYIIni7OODdqBYAgFmDWkqPxc3tj7BF+7W+pthGhFd6BOP7w0l6i5MMS3lfnXJVR4WpsmR3osK+Gf/NMD3zz3PYOqWbwr3U9T978cdTKo+tj03WKKby+1V+HvrNEQDAzft50n3/JmagoVcdvPlbPOYOCcGgWjBRKJvAiIhqQLYJrrpmtVb+bnLbmi7X8VbfplrHReYpMS1X5bGCYuULiapaIsTQEtNzpJ+nbT6L136Jw51Hj/H6hjMmiUffmAARkVUyxKgX2YSmpZ+rwvH+IT4aX2vmwBbSzxU1T2Ra+uiAfFbNfDtFpRKkZxcCkB9t9tc51c1YNVFdH6CqFU8lahZvtURMgIjIKtVwrVQ5c4eEAACWjm4n3adJnxB1Gnk51+h8sjxJmfnouvgfHLuRpXbts+rcflig9cKtypr3qvY9qphdu7ZgHyAiskpVm6NqYlKPxniucxCcHWzx9qaz0v2bXu2KZ787XllQTdY1pK0/dp5Pk24PCvXDrEEttZ7sjizfc98fr76QGpN/jcOFOznVF5RR8Wpm5hVJ99XGjs+yWANERFZl59TueK1nY3zwX60NALT5by4hXzfdF0B1rrKemSAAXRt7IcizjnSfQvojs2PFcx1w4oN+GBMeiG1TIiESiTC5VxN0beyl0f0Pz+wjt3110SDVsdqLMaC1ZgvDknnRpGZR2+QHKG8STridjXCZDv3q5kSqzvGbmnUKNyUmQERkVVoHuGP24BC4ySz++N34MEzqHow/Xos06L3VNbuJRCL4ujni01Ft0SGorsLxQaF+aq8dKJNoAernpRnZsYH6QMls1bRpVdbl9MoO2dmPi/HbSfk108rUjD57ef0ptYvKrvj3es0DNDAmQERk9fzdnTB3aCsEedWpvrCGKn46KobMj49oqNDxemDr8qRGk5qnz59phy+eaQePOpqv2t0hyEPpfhuR6sn1yLypSzpqYn9ihlzzF1C50Kky/17OUNs5uya1R8bCPkBERNXQpcN0xSiewW38ETe3Pzyd7RFz9b5cmQ5BdfHvu73g5+5Y7fVcHGwxKqwBVh64rvaHSRMikUgvi3RS7RJ96V71hWTM3pag8pi62iNzwRogIiI9GtK2fIK413tXrp3k5eIAkUiEXs3rYdXzHfHPu72kxxrXc0Ede83/LaosGftwaCsAms8XZCcWWcS/0Mm8qauNKrWAIfOsASIi0qOvn+2AWQNbKvTJAcprXgbXcAZdcZUM6PTc/vD+bzJGbRKpF7o2xIEr96svSKQDS0iwWQNERKRHYhuR0uRHX6rOHu1oV7mqfLsG7irPc3WsTI4kAtAvxBcH3+utl5gm96p9K4VTzRSXMQEiIiI9sqkyCkh2K7KpN74bF4Z/ZZrYlKkYAdfIWz+TLcqujUYEAEUlypf1MCdMgIiILEjVUdBV+wRFtfZD43qKK4rLmtQjWPp58cg2csc+GdEGp+b0x8TuwVVPI9LYYyZARESWqyIJmD04pJqSxtOpkafcti5rmslO2vhc5yC5OYbGdglCPVcH2Iorr1s1SZL1Sg8mSqRIk5GNpsYEiIhIhQ+HtkLC/Cj0al7P1KFIzRzYAs92CpRuqxui7+Vsr9E1Fz0VihcjG2H32z0qryuTWD3XOUjluXOGtNLoHtrq2tiz+kJktvqHmP9M40yAiIjUcHXUfOJBY6hjb6txp+OFT4WiW1MvfDcurHJmRiW8XBww/8nWCJFZH62RHieF1MX6lzqb9P5UM5YwzxSHwRMRWTB1NUD+7k7YMKnrf1vntLruqLAGuPvoMbo28ZJu/xl3W6cYt06JRCt/N7T8cI/G58iObiPLYwGj4FkDRERkyXTpA6QJW7ENpke1QGQTbwDA56Pa4uScfnJl5g7RrG9Ux6C6WiU0g9uoX/eMzB/XAiMiIrMgFtcsURKJRPBxle/YWrdO9X2M3otqrvW9Ph/VTutzyLwUW8BM0EyAiIgsjC6tCz9MCIe3iz2WP9te3+GoveebfZtpXH5IW3/sn95TbpSaMgufCq1paERMgIiILI2nzOgucdWJgVQIa+iJU3P6Y3j7+jW6d5v6lbNNV5eIabuI7NS+zdDUx7XacpqOblPn8Mw+eK1XY7Vl2gd61Pg+ZL6YABERWRh3JztsmxKJv9/qrnECBJQ3Y9XUple7Kt3fwlcxcdGmf1Irfzc09VE/gaMm/N0d0SHIA+fnRyk9fnJOP+yf3guBnnXwZh/1i8d+/WyHGsdD5oujwIiILFCHoLomua+q5qmtUyJx9V4u/j6fhh+OJFV7nf3Te+GPuFR8G3MTAPD3W90VlvmoMKJDfWyLvyPdVjfCetOrXRHkWUdlsufj6oiKSiZlUxy4Odoip7AUAGDDKoJajf/zEhFRjTk72KJDUF25kWGCikayQaF+aOrjgk4NKyc7VJX8AIpzygR4OGJMeKDSsiKIVCY//Vr6KOzbObW73PZvr3TF9je64eisvgoLz1LtwhogIiLSiZ2SkWWaNLO1+m/Cxb4tfTCpezDaqFnFHgDCGnliQmQjjFpzDC90CUKHoLpo28ADm0+nahRn50aeeH9QC7QOULxP1X2hMn2c7uUUanT95c+2x9ubzmpUlswHEyAiItLKlN5NcCr5AQaG1my+HhsbEeYOVb2UxsH3euNk8gM83bEBxDYi3PhksPSYqr5Pbk6KP2s2NuWdwFUJ8qyDlAcFcHWUP9dcaoAS5kehzfx9pg6j1mECREREWpk5sGWNztd0GH8jb2c08nZWebyi5uWrMe3g7eKA4lIJPDSYm6iqn1/ujM/2XsaU3vKdojXpYD4mPFCu1qjCyI71sePsXZTqYUpkc1uOpbZgAkRERBZpePv6GBTqD3vbmnVnbeTtjFXPhynsl81/vnimHbbF38a0/s1hayPCiFWxAIDXezdRmqQtHd0eU3o3Qf+lh2oUGxkOEyAiIjIqfa6TWdPkRx3ZjtmdGtXFqLAGAID8olKFsm3quyPhTjYA2dmvzaMJjZQz+SiwVatWITg4GI6OjggLC8Phw4dVlt26dSueeOIJ1KtXD25uboiIiMDevXuNGC0REWlKWdMQoHp0mLkRy/QBUtWSVdFPSLa2SJvZr8l0TJoAbd68GdOmTcOcOXMQHx+PHj16YNCgQUhJSVFa/tChQ3jiiSewa9cuxMXFoU+fPhg2bBji4+ONHDkREalybl4UYmf1VVg7rEIjL9X9esyJbCdo2aH4sn2jpZ+VdJhW1Yfa313+uQxrFyC3/ftrEQrnVDdrtTlp5FUHK8d2NHUY1TJpArR06VJMnDgRkyZNQkhICJYtW4bAwECsXr1aaflly5Zh5syZ6NSpE5o1a4ZPPvkEzZo1w19//aXyHkVFRcjJyZH7IyIiw3F3skOAh5PC/s2vdsWMAS3wZJUffEOp/18Mg0L9dTpfdiJEXfoyq2rqs60yfcAHgys7lb8/sCU6B3ti4ytd4e/uiDUvlPdNaqOiNq2qBnUVn7uxHZzRB0Pa6vbMjclkfYCKi4sRFxeHWbNmye2PiopCbGysRteQSCTIzc2Fp6fq4Y2LFy/GggULahQrERHVXJfGXujS2Mto9/vrre44d/sRejarp9P5YhU1QLIqimja2+eFrkG4nJaL1AePpfv83Z2wbUok3J3s0Lhe+XIgEU28cGx2P2mZwaH+mDmwAB0C6+K5748rvfaaF8Jw7EYmfjp2S8NorJvJaoAyMzNRVlYGX19fuf2+vr5IT0/X6Bpffvkl8vPzMXr0aJVlZs+ejezsbOlfaqpmE2cREZFl83S2R58WPlqtlyZLrglMZr+yXEhZc5eyfYueaqN0f4egutLkR2ksNiJM6d0UEU1UJ5ADQ/3k4lRXGzQhoqHKYwAQ4O6I/dN7qS1j6UzeCbrqrKGCIGg0k+jGjRsxf/58bN68GT4+itObV3BwcICbm5vcHxERUXVkR4G5yczFI5tQOdiKAQDO9ooNKt7ODtLPXz/XAeteDAcgv0js6bn99RcwIJ2c0t/dUW0C1L2aWrGuTbzQ1McFS0e302t85sRkTWDe3t4Qi8UKtT0ZGRkKtUJVbd68GRMnTsQff/yB/v31+/IQERFV+GFCOPKKSuEn03HZ0U6MGQNaoLhUgnqu5UnOoqdCMenn03itZ2VnZfc6dtjyegTsxWL55T5k/o3v7VKZJOlDZBNv7H67BxrUdcIrP59WWsZOLEKTeuo7or/eqwkAYGTHBgit746or2rffEYmS4Ds7e0RFhaG6OhojBgxQro/Ojoaw4cPV3nexo0b8fLLL2Pjxo0YMmSIMUIlIiIr1S9E+T/I3+gjP2t0I29npU1GypbgaObjgpNJD/QToBIh/qpbOqY/0Rxv9W2K5KwClWUuLxwIRzuxdNvRVqy0XON6zrh5P1/3QE3MpBMhTp8+HePGjUN4eDgiIiLw3XffISUlBZMnTwZQ3n/nzp07+PnnnwGUJz/jx4/H8uXL0bVrV2ntkZOTE9zdNeshT0REZErvD2oJsY0Iw9vX19s1Zw5soVG5fiE+EIlE8HNTPkUBALnkB1A9nF+fE1qagkkToDFjxiArKwsff/wx0tLSEBoail27dqFhw/LOWWlpaXJzAn377bcoLS3FG2+8gTfeeEO6f8KECVi/fr2xwyciItKam6MdPh4eqvP57Rq449ztbOn2nmk90NJPs/6tFUmLk70Yp+b0x+PiMly8m40b9/Pwxb6r1Z7/9XMdMHVj+dx7BcWKM2JbEpMvhTFlyhRMmTJF6bGqSc3BgwcNHxAREZEZ2/RqBK5l5OLJFUcBqF61fvagEAxfeRT1PZxw51H5sHvZWpuK/ktBXnWw9vBNlfdrUNcJEY294GQvRnjDutL9Lf3ccC/nvlzZV3oE6/SdTMHkCRARERFpzslerNHEiO0CPXBl0UDYi20QPHsXANVD48d0CsSGEyno11JxVLVIJMLGV7tKt3e/3QPuTnYQ24jwbcxNtPRzxa4LaRjRoT6GtDH/CRAriARVszvVUjk5OXB3d0d2djaHxBMRkcWatike9/OK8MvLXeSG7CuT+qAAj0vK0NzX1UjR6Z++f79ZA0RERGSBlj3bQeOygZ51DBiJZTL5RIhERERExsYEiIiIiKwOEyAiIiKyOkyAiIiIyOowASIiIiKrwwSIiIiIrA4TICIiIrI6TICIiIjI6jABIiIiIqvDBIiIiIisDhMgIiIisjpMgIiIiMjqMAEiIiIiq8MEiIiIiKyOrakDMDZBEAAAOTk5Jo6EiIiINFXxu13xO15TVpcA5ebmAgACAwNNHAkRERFpKzc3F+7u7jW+jkjQVyplISQSCe7evQtXV1eIRCK9XjsnJweBgYFITU2Fm5ubXq9tSfgcKvFZlONzKMfnUInPohyfQzlNnoMgCMjNzUVAQABsbGreg8fqaoBsbGzQoEEDg97Dzc3Nql/kCnwOlfgsyvE5lONzqMRnUY7PoVx1z0EfNT8V2AmaiIiIrA4TICIiIrI6TID0yMHBAfPmzYODg4OpQzEpPodKfBbl+BzK8TlU4rMox+dQzhTPweo6QRMRERGxBoiIiIisDhMgIiIisjpMgIiIiMjqMAEiIiIiq8MESE9WrVqF4OBgODo6IiwsDIcPHzZ1SHo1f/58iEQiuT8/Pz/pcUEQMH/+fAQEBMDJyQm9e/fGxYsX5a5RVFSEt956C97e3nB2dsaTTz6J27dvG/uraOXQoUMYNmwYAgICIBKJsH37drnj+vreDx8+xLhx4+Du7g53d3eMGzcOjx49MvC30051z+LFF19UeEe6du0qV6Y2PIvFixejU6dOcHV1hY+PD5566ilcuXJFrow1vBeaPAdreCdWr16Ntm3bSifwi4iIwO7du6XHreFdqFDdszC790GgGtu0aZNgZ2cnfP/998KlS5eEt99+W3B2dhZu3bpl6tD0Zt68eULr1q2FtLQ06V9GRob0+JIlSwRXV1dhy5YtQkJCgjBmzBjB399fyMnJkZaZPHmyUL9+fSE6Olo4c+aM0KdPH6Fdu3ZCaWmpKb6SRnbt2iXMmTNH2LJliwBA2LZtm9xxfX3vgQMHCqGhoUJsbKwQGxsrhIaGCkOHDjXW19RIdc9iwoQJwsCBA+XekaysLLkyteFZDBgwQPjxxx+FCxcuCGfPnhWGDBkiBAUFCXl5edIy1vBeaPIcrOGd2LFjh7Bz507hypUrwpUrV4QPPvhAsLOzEy5cuCAIgnW8CxWqexbm9j4wAdKDzp07C5MnT5bb17JlS2HWrFkmikj/5s2bJ7Rr107pMYlEIvj5+QlLliyR7issLBTc3d2FNWvWCIIgCI8ePRLs7OyETZs2ScvcuXNHsLGxEfbs2WPQ2PWl6o++vr73pUuXBADC8ePHpWWOHTsmABAuX75s4G+lG1UJ0PDhw1WeU1ufRUZGhgBAiImJEQTBet+Lqs9BEKz3nahbt66wdu1aq30XZFU8C0Ewv/eBTWA1VFxcjLi4OERFRcntj4qKQmxsrImiMoxr164hICAAwcHBePbZZ3Hz5k0AQFJSEtLT0+WegYODA3r16iV9BnFxcSgpKZErExAQgNDQUIt9Tvr63seOHYO7uzu6dOkiLdO1a1e4u7tb3LM5ePAgfHx80Lx5c7zyyivIyMiQHqutzyI7OxsA4OnpCcB634uqz6GCNb0TZWVl2LRpE/Lz8xEREWG17wKg+CwqmNP7YHWLoepbZmYmysrK4OvrK7ff19cX6enpJopK/7p06YKff/4ZzZs3x71797Bo0SJERkbi4sWL0u+p7BncunULAJCeng57e3vUrVtXoYylPid9fe/09HT4+PgoXN/Hx8eins2gQYPwzDPPoGHDhkhKSsKHH36Ivn37Ii4uDg4ODrXyWQiCgOnTp6N79+4IDQ0FYJ3vhbLnAFjPO5GQkICIiAgUFhbCxcUF27ZtQ6tWraQ/yNb0Lqh6FoD5vQ9MgPREJBLJbQuCoLDPkg0aNEj6uU2bNoiIiECTJk3w008/STux6fIMasNz0sf3Vlbe0p7NmDFjpJ9DQ0MRHh6Ohg0bYufOnRg5cqTK8yz5Wbz55ps4f/48jhw5onDMmt4LVc/BWt6JFi1a4OzZs3j06BG2bNmCCRMmICYmRnrcmt4FVc+iVatWZvc+sAmshry9vSEWixUyz4yMDIWsvzZxdnZGmzZtcO3aNeloMHXPwM/PD8XFxXj48KHKMpZGX9/bz88P9+7dU7j+/fv3LfbZAIC/vz8aNmyIa9euAah9z+Ktt97Cjh07cODAATRo0EC639reC1XPQZna+k7Y29ujadOmCA8Px+LFi9GuXTssX77c6t4FQPWzUMbU7wMToBqyt7dHWFgYoqOj5fZHR0cjMjLSRFEZXlFRERITE+Hv74/g4GD4+fnJPYPi4mLExMRIn0FYWBjs7OzkyqSlpeHChQsW+5z09b0jIiKQnZ2NkydPSsucOHEC2dnZFvtsACArKwupqanw9/cHUHuehSAIePPNN7F161b8+++/CA4OljtuLe9Fdc9Bmdr6TlQlCAKKioqs5l1Qp+JZKGPy90GrLtOkVMUw+B9++EG4dOmSMG3aNMHZ2VlITk42dWh68+677woHDx4Ubt68KRw/flwYOnSo4OrqKv2OS5YsEdzd3YWtW7cKCQkJwnPPPad0qGeDBg2E/fv3C2fOnBH69u1r9sPgc3Nzhfj4eCE+Pl4AICxdulSIj4+XTnGgr+89cOBAoW3btsKxY8eEY8eOCW3atDG7Ia7qnkVubq7w7rvvCrGxsUJSUpJw4MABISIiQqhfv36texavv/664O7uLhw8eFBuOG9BQYG0jDW8F9U9B2t5J2bPni0cOnRISEpKEs6fPy988MEHgo2NjbBv3z5BEKzjXaig7lmY4/vABEhPVq5cKTRs2FCwt7cXOnbsKDcUtDaomLvCzs5OCAgIEEaOHClcvHhRelwikQjz5s0T/Pz8BAcHB6Fnz55CQkKC3DUeP34svPnmm4Knp6fg5OQkDB06VEhJSTH2V9HKgQMHBAAKfxMmTBAEQX/fOysrS3j++ecFV1dXwdXVVXj++eeFhw8fGulbakbdsygoKBCioqKEevXqCXZ2dkJQUJAwYcIEhe9ZG56FsmcAQPjxxx+lZazhvajuOVjLO/Hyyy9L/9tfr149oV+/ftLkRxCs412ooO5ZmOP7IBIEQdCuzoiIiIjIsrEPEBEREVkdJkBERERkdZgAERERkdVhAkRERERWhwkQERERWR0mQERERGR1mAARERGR1WECRERERFaHCRARGUyjRo2wbNkyjcsfPHgQIpEIjx49MlhM5kTb50NE+mNr6gCIyHz07t0b7du319uP8qlTp+Ds7Kxx+cjISKSlpcHd3V0v9yciUoUJEBFpRRAElJWVwda2+v981KtXT6tr29vbw8/PT9fQiIg0xiYwIgIAvPjii4iJicHy5cshEokgEomQnJwsbZbau3cvwsPD4eDggMOHD+PGjRsYPnw4fH194eLigk6dOmH//v1y16zaxCMSibB27VqMGDECderUQbNmzbBjxw7p8apNYOvXr4eHhwf27t2LkJAQuLi4YODAgUhLS5OeU1paiqlTp8LDwwNeXl54//33MWHCBDz11FNqv29sbCx69uwJJycnBAYGYurUqcjPz5eLfeHChRg7dixcXFwQEBCAb775Ru4aKSkpGD58OFxcXODm5obRo0fj3r17cmV27NiB8PBwODo6wtvbGyNHjpQ7XlBQgJdffhmurq4ICgrCd999pzZuItIPJkBEBABYvnw5IiIi8MorryAtLQ1paWkIDAyUHp85cyYWL16MxMREtG3bFnl5eRg8eDD279+P+Ph4DBgwAMOGDUNKSora+yxYsACjR4/G+fPnMXjwYDz//PN48OCByvIFBQX44osv8Msvv+DQoUNISUnBe++9Jz3+6aefYsOGDfjxxx9x9OhR5OTkYPv27WpjSEhIwIABAzBy5EicP38emzdvxpEjR/Dmm2/Klfv888/Rtm1bnDlzBrNnz8Y777yD6OhoAOU1YU899RQePHiAmJgYREdH48aNGxgzZoz0/J07d2LkyJEYMmQI4uPj8c8//yA8PFzuHl9++SXCw8MRHx+PKVOm4PXXX8fly5fVxk9EeqDTmvdEVCv16tVLePvtt+X2HThwQAAgbN++vdrzW7VqJXzzzTfS7YYNGwpfffWVdBuAMHfuXOl2Xl6eIBKJhN27d8vd6+HDh4IgCMKPP/4oABCuX78uPWflypWCr6+vdNvX11f4/PPPpdulpaVCUFCQMHz4cJVxjhs3Tnj11Vfl9h0+fFiwsbERHj9+LI194MCBcmXGjBkjDBo0SBAEQdi3b58gFouFlJQU6fGLFy8KAISTJ08KgiAIERERwvPPP68yjoYNGwovvPCCdFsikQg+Pj7C6tWrVZ5DRPrBGiAi0kjVmov8/HzMnDkTrVq1goeHB1xcXHD58uVqa4Datm0r/ezs7AxXV1dkZGSoLF+nTh00adJEuu3v7y8tn52djXv37qFz587S42KxGGFhYWpjiIuLw/r16+Hi4iL9GzBgACQSCZKSkqTlIiIi5M6LiIhAYmIiACAxMRGBgYFytWQVz6KizNmzZ9GvXz+1scg+D5FIBD8/P7XPg4j0g52giUgjVUdzzZgxA3v37sUXX3yBpk2bwsnJCaNGjUJxcbHa69jZ2clti0QiSCQSrcoLgqCwT1bV41VJJBK89tprmDp1qsKxoKAgtedW3EsQBIX7Vt3v5OSk9lqA9s+DiPSDNUBEJGVvb4+ysjKNyh4+fBgvvvgiRowYgTZt2sDPzw/JycmGDbAKd3d3+Pr64uTJk9J9ZWVliI+PV3tex44dcfHiRTRt2lThz97eXlru+PHjcucdP34cLVu2BFBe25OSkoLU1FTp8UuXLiE7OxshISEAymt3/vnnnxp/TyLSP9YAEZFUo0aNcOLECSQnJ8PFxQWenp4qyzZt2hRbt27FsGHDIBKJ8OGHH5qk5uKtt97C4sWL0bRpU7Rs2RLffPMNHj58qLR2psL777+Prl274o033sArr7wCZ2dnJCYmIjo6Wm6k19GjR/HZZ5/hqaeeQnR0NP744w/s3LkTANC/f3+0bdsWzz//PJYtW4bS0lJMmTIFvXr1kjYXzps3D/369UOTJk3w7LPPorS0FLt378bMmTMN+1CIqFqsASIiqffeew9isRitWrVCvXr11Pbn+eqrr1C3bl1ERkZi2LBhGDBgADp27GjEaMu9//77eO655zB+/HhERERI+/M4OjqqPKdt27aIiYnBtWvX0KNHD3To0AEffvgh/P395cq9++67iIuLQ4cOHbBw4UJ8+eWXGDBgAIDypqrt27ejbt266NmzJ/r374/GjRtj8+bN0vN79+6NP/74Azt27ED79u3Rt29fnDhxwjAPgoi0IhKqaywnIrIgEokEISEhGD16NBYuXKjzdRo1aoRp06Zh2rRp+guOiMwGm8CIyKLdunUL+/btQ69evVBUVIQVK1YgKSkJY8eONXVoRGTG2ARGRBbNxsYG69evR6dOndCtWzckJCRg//790o7IRETKsAmMiIiIrA5rgIiIiMjqMAEiIiIiq8MEiIiIiKwOEyAiIiKyOkyAiIiIyOowASIiIiKrwwSIiIiIrA4TICIiIrI6/w+1Vrw53YqxbQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "eval_loss = 0.2869\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "\n",
    "class LR(nn.Module):\n",
    "    def __init__(self, input_dim, output_dim):\n",
    "        super(LR, self).__init__()\n",
    "        self.linear = nn.Linear(input_dim, output_dim)\n",
    "        \n",
    "    def forward(self, input_feats, labels=None):\n",
    "        outputs = self.linear(input_feats)\n",
    "        \n",
    "        if labels is not None:\n",
    "            loss_fc = nn.CrossEntropyLoss()\n",
    "            loss = loss_fc(outputs, labels)\n",
    "            return (loss, outputs)\n",
    "        \n",
    "        return outputs\n",
    "\n",
    "model = LR(len(dataset.token2id), len(dataset.label2id))\n",
    "\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torch.optim import SGD, Adam\n",
    "\n",
    "# 使用PyTorch的DataLoader来进行数据循环，因此按照PyTorch的接口\n",
    "# 实现myDataset和DataCollator两个类\n",
    "# myDataset是对特征向量和标签的简单封装便于对齐接口，\n",
    "# DataCollator用于批量将数据转化为PyTorch支持的张量类型\n",
    "class myDataset(Dataset):\n",
    "    def __init__(self, X, Y):\n",
    "        self.X = X\n",
    "        self.Y = Y\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.X)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        return (self.X[idx], self.Y[idx])\n",
    "\n",
    "class DataCollator:\n",
    "    @classmethod\n",
    "    def collate_batch(cls, batch):\n",
    "        feats, labels = [], []\n",
    "        for x, y in batch:\n",
    "            feats.append(x)\n",
    "            labels.append(y)\n",
    "        # 直接将一个ndarray的列表转化为张量是非常慢的，\n",
    "        # 所以需要提前将列表转化为一整个ndarray\n",
    "        feats = torch.tensor(np.array(feats), dtype=torch.float)\n",
    "        labels = torch.tensor(np.array(labels), dtype=torch.long)\n",
    "        return {'input_feats': feats, 'labels': labels}\n",
    "\n",
    "# 设置训练超参数和优化器，模型初始化\n",
    "epochs = 50\n",
    "batch_size = 128\n",
    "learning_rate = 1e-3\n",
    "weight_decay = 0\n",
    "\n",
    "train_dataset = myDataset(train_F, train_Y)\n",
    "test_dataset = myDataset(test_F, test_Y)\n",
    "\n",
    "data_collator = DataCollator()\n",
    "train_dataloader = DataLoader(train_dataset, batch_size=batch_size,\\\n",
    "    shuffle=True, collate_fn=data_collator.collate_batch)\n",
    "test_dataloader = DataLoader(test_dataset, batch_size=batch_size,\\\n",
    "    shuffle=False, collate_fn=data_collator.collate_batch)\n",
    "optimizer = Adam(model.parameters(), lr=learning_rate,\\\n",
    "    weight_decay=weight_decay)\n",
    "model.zero_grad()\n",
    "model.train()\n",
    "\n",
    "from tqdm import tqdm, trange\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# 模型训练\n",
    "with trange(epochs, desc='epoch', ncols=60) as pbar:\n",
    "    epoch_loss = []\n",
    "    for epoch in pbar:\n",
    "        model.train()\n",
    "        for step, batch in enumerate(train_dataloader):\n",
    "            loss = model(**batch)[0]\n",
    "            pbar.set_description(f'epoch-{epoch}, loss={loss.item():.4f}')\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            model.zero_grad()\n",
    "            epoch_loss.append(loss.item())\n",
    "\n",
    "    epoch_loss = np.array(epoch_loss)\n",
    "    # 打印损失曲线\n",
    "    plt.plot(range(len(epoch_loss)), epoch_loss)\n",
    "    plt.xlabel('training epoch')\n",
    "    plt.ylabel('loss')\n",
    "    plt.show()\n",
    "    \n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        loss_terms = []\n",
    "        for batch in test_dataloader:\n",
    "            loss = model(**batch)[0]\n",
    "            loss_terms.append(loss.item())\n",
    "        print(f'eval_loss = {np.mean(loss_terms):.4f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "10808854",
   "metadata": {},
   "source": [
    "下面的代码使用训练好的模型对测试集进行预测，并报告分类结果。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "11a9bf62",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test example-0, prediction = 0, label = 0\n",
      "test example-1, prediction = 0, label = 0\n",
      "test example-2, prediction = 1, label = 1\n",
      "test example-3, prediction = 1, label = 1\n",
      "test example-4, prediction = 1, label = 1\n"
     ]
    }
   ],
   "source": [
    "LR_preds = []\n",
    "model.eval()\n",
    "for batch in test_dataloader:\n",
    "    with torch.no_grad():\n",
    "        _, preds = model(**batch)\n",
    "        preds = np.argmax(preds, axis=1)\n",
    "        LR_preds.extend(preds)\n",
    "            \n",
    "for i, (p, y) in enumerate(zip(LR_preds, test_Y)):\n",
    "    if i >= 5:\n",
    "        break\n",
    "    print(f'test example-{i}, prediction = {p}, label = {y}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c5feb65e",
   "metadata": {},
   "source": [
    "下面的代码展示多分类情况下宏平均和微平均的算法。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a5ac32c5",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NB: micro-f1 = 0.8961520630505331, macro-f1 = 0.8948572078813896\n",
      "LR: micro-f1 = 0.914696337505795, macro-f1 = 0.9139442964719672\n"
     ]
    }
   ],
   "source": [
    "test_Y = np.array(test_Y)\n",
    "NB_preds = np.array(NB_preds)\n",
    "LR_preds = np.array(LR_preds)\n",
    "\n",
    "def micro_f1(preds, labels):\n",
    "    TP = np.sum(preds == labels)\n",
    "    FN = FP = 0\n",
    "    for i in range(len(dataset.label2id)):\n",
    "        FN += np.sum((preds == i) & (labels != i))\n",
    "        FP += np.sum((preds != i) & (labels == i))\n",
    "    precision = TP / (TP + FP)\n",
    "    recall = TP / (TP + FN)\n",
    "    f1 = 2 * precision * recall / (precision + recall)\n",
    "    return f1\n",
    "\n",
    "def macro_f1(preds, labels):\n",
    "    f_scores = []\n",
    "    for i in range(len(dataset.label2id)):\n",
    "        TP = np.sum((preds == i) & (labels == i))\n",
    "        FN = np.sum((preds == i) & (labels != i))\n",
    "        FP = np.sum((preds != i) & (labels == i))\n",
    "        precision = TP / (TP + FP)\n",
    "        recall = TP / (TP + FN)\n",
    "        f1 = 2 * precision * recall / (precision + recall)\n",
    "        f_scores.append(f1)\n",
    "    return np.mean(f_scores)\n",
    "\n",
    "print(f'NB: micro-f1 = {micro_f1(NB_preds, test_Y)}, '+\\\n",
    "      f'macro-f1 = {macro_f1(NB_preds, test_Y)}')\n",
    "print(f'LR: micro-f1 = {micro_f1(LR_preds, test_Y)}, '+\\\n",
    "      f'macro-f1 = {macro_f1(LR_preds, test_Y)}')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.17"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}