BERT_Fine_Tuning_Tutorial_with_PyTorch_(updated Feb 24).ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "5q2LY1BjyNVQ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 180
        },
        "outputId": "466df093-0d16-42aa-bcb2-0653f6259c4d"
      },
      "outputs": [
        {
          "output_type": "error",
          "ename": "SystemError",
          "evalue": "GPU device not found",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mSystemError\u001b[0m                               Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-3-cd5bb073bb20>\u001b[0m in \u001b[0;36m<cell line: 7>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Found GPU at: {}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m     \u001b[0;32mraise\u001b[0m \u001b[0mSystemError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'GPU device not found'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mSystemError\u001b[0m: GPU device not found"
          ]
        }
      ],
      "source": [
        "import tensorflow as tf\n",
        "\n",
        "# Get the GPU device name.\n",
        "device_name = tf.test.gpu_device_name()\n",
        "\n",
        "# The device name should look like the following:\n",
        "if device_name == '/device:GPU:0':\n",
        "    print('Found GPU at: {}'.format(device_name))\n",
        "else:\n",
        "    raise SystemError('GPU device not found')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "w71JPK6By1Um",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "47dce086-e2e9-45b8-d6a7-aa7603023e56"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "No GPU available, using the CPU instead.\n"
          ]
        }
      ],
      "source": [
        "import torch\n",
        "\n",
        "# If there's a GPU available...\n",
        "if torch.cuda.is_available():\n",
        "\n",
        "    # Tell PyTorch to use the GPU.\n",
        "    device = torch.device(\"cuda\")\n",
        "\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "\n",
        "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
        "\n",
        "# If not...\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "WvvABQDT1k4U",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "896d9090-403d-4a72-ba6b-87f03afddb88"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.37.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.13.1)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.20.3)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.25.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.12.25)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.15.2)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.2)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.2)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.19.3->transformers) (2023.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.19.3->transformers) (4.9.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2024.2.2)\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "C_HpgToU10Fd"
      },
      "outputs": [],
      "source": [
        "##!pip install wget"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "9Y1Sfyhw15Cx"
      },
      "outputs": [],
      "source": [
        "##import wget\n",
        "##import os\n",
        "\n",
        "##print('Downloading dataset...')\n",
        "\n",
        "# The URL for the dataset zip file.\n",
        "##url = 'https://nyu-mll.github.io/CoLA/cola_public_1.1.zip'\n",
        "\n",
        "# Download the file (if we haven't already)\n",
        "##if not os.path.exists('./cola_public_1.1.zip'):\n",
        "##    wget.download(url, './cola_public_1.1.zip')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "gZ84CM3Q2P7I"
      },
      "outputs": [],
      "source": [
        "# Unzip the dataset (if we haven't already)\n",
        "##if not os.path.exists('./cola_public/'):\n",
        "##    !unzip cola_public_1.1.zip"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "UoxEXrZsbagH",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "43300f17-20f6-447b-9e82-4158eff173cd"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of training sentences: 7,204\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     post_number                                          title  \\\n",
              "4401     3527559            以 巴 大戰 蕭 若 元 大爆 以色列 軍隊 攻入 加 莎 終極 策略   \n",
              "4051     3411216  歲 中年 父親 陳 悅 晉 排除萬難 加入 警隊 加入 警隊 的 初心 是 止 暴 制 亂   \n",
              "680       101688                                      弘 豬 天 一 食   \n",
              "57        100128                                  推 完場 曼聯 對 韋 根   \n",
              "2757      800010                                     同一 齊 學 貓 叫   \n",
              "1398      103447                                     後 回憶 唱 唱咩歌   \n",
              "5052     3579611                                         究竟 請 請   \n",
              "3539     1001821                                       唔知 蔥蔥 開心   \n",
              "1681      104126                                          寫 寫 死   \n",
              "182       100483                                 求 鄭中基 今晚 單 頭 飛   \n",
              "\n",
              "                                           main_content  \\\n",
              "4401  燒 山 係咪 放 左 針 以軍 參謀長 蕭 若 元 依家 好多 軍事 人人 仲 發 緊 夢 ...   \n",
              "4051  歲 的 陳 悅 晉 剛 於 警察 學院 畢業 與 妻子 育 一 子 一 女 投身 警隊 前 ...   \n",
              "680                          韓國 野 或 日本 野 人 想見 著 黑 衫 牛仔褲   \n",
              "57                                                  NaN   \n",
              "2757                                       一齊 喵 喵 喵 喵 喵   \n",
              "1398  唱 好多 歌 活 著 失戀 王 好心 報 最後 祝福 無賴 開始 戀愛 好多 識 唱 宜家 ...   \n",
              "5052            見工 時 好似 輕鬆 又帶 參觀 下個 以為 有機 點知 左 個半 禮拜 消息   \n",
              "3539                憂心 蔥蔥 魷魚 章魚 墨魚 三 打工 中途 休息 先 魷魚 魷 落場   \n",
              "1681      聽日 仲未 寫 就係 一睇 明 題目 偏門 搵 類似 字 寫 左 野 寫 係咪 玩完 收皮   \n",
              "182                                                 NaN   \n",
              "\n",
              "                                               comments word_count  \\\n",
              "4401  順民 馬 居士 連 下 四 城 打氣 區分 享 自 討論區 最後 鏟平 蕭 生 釋 條針 放...        290   \n",
              "4051  年 黑 暴 肆虐 暴徒 四處 破壞 摧毀 其 安穩 的 生活 年初 他 因公 司 經營 困難...        568   \n",
              "680   變 左 沐 八 既 人 約 左 巴打 天水圍 飽 屙 尋晚 唸 住 一個 韓燒 去到 尖咀 ...         96   \n",
              "57                                        乜事 無 曬 人 靚 月巴          6   \n",
              "2757  愛心 和 那個 了 雙馬尾 主要 教人 養 貓 狗 幫 聯絡 下 樓主 射精 了 隻 貓 舐...         46   \n",
              "1398  最後 祝福 首 有得 唱 好似 套 劇 出 左 冇耐 最後 祝福 首 有得 唱 好似 套 劇...        159   \n",
              "5052  移民 細 公司 見 老闆 當住 請 努力 搵 啦 見 完工 不嬲 諗住 見 見工 時 好似 ...         38   \n",
              "3539  魷 樂 場 好笑 係唔係 開 對 衝擊 太大 先 呢啲 唉 唔好 提 喇 先 開 如 如 如...         55   \n",
              "1681  係唔係 寫 漏 聽日 仲未 寫 就係 一睇 明 題目 偏門 搵 類似 字 寫 左 野 寫 係...         93   \n",
              "182   一個人 演唱會 單張 幫 幫手 俾 忍心 見到 一個人 戇鳩鳩 其實 真心 粉絲 今日 打算...        100   \n",
              "\n",
              "                                          URL Category Classification  \n",
              "4401  https://lihkg.com/thread/3527559/page/1      5.0            時事台  \n",
              "4051  https://lihkg.com/thread/3411216/page/1      5.0            時事台  \n",
              "680    https://lihkg.com/thread/101688/page/1     16.0            飲食台  \n",
              "57     https://lihkg.com/thread/100128/page/1      6.0            體育台  \n",
              "2757   https://lihkg.com/thread/800010/page/1      7.0            娛樂台  \n",
              "1398   https://lihkg.com/thread/103447/page/1      7.0            娛樂台  \n",
              "5052  https://lihkg.com/thread/3579611/page/1     14.0            上班台  \n",
              "3539  https://lihkg.com/thread/1001821/page/1     31.0            創意台  \n",
              "1681   https://lihkg.com/thread/104126/page/1     18.0            學術台  \n",
              "182    https://lihkg.com/thread/100483/page/1      7.0            娛樂台  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6ff5bac5-8b54-4ad2-82f9-2a127c64314b\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>post_number</th>\n",
              "      <th>title</th>\n",
              "      <th>main_content</th>\n",
              "      <th>comments</th>\n",
              "      <th>word_count</th>\n",
              "      <th>URL</th>\n",
              "      <th>Category</th>\n",
              "      <th>Classification</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>4401</th>\n",
              "      <td>3527559</td>\n",
              "      <td>以 巴 大戰 蕭 若 元 大爆 以色列 軍隊 攻入 加 莎 終極 策略</td>\n",
              "      <td>燒 山 係咪 放 左 針 以軍 參謀長 蕭 若 元 依家 好多 軍事 人人 仲 發 緊 夢 ...</td>\n",
              "      <td>順民 馬 居士 連 下 四 城 打氣 區分 享 自 討論區 最後 鏟平 蕭 生 釋 條針 放...</td>\n",
              "      <td>290</td>\n",
              "      <td>https://lihkg.com/thread/3527559/page/1</td>\n",
              "      <td>5.0</td>\n",
              "      <td>時事台</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4051</th>\n",
              "      <td>3411216</td>\n",
              "      <td>歲 中年 父親 陳 悅 晉 排除萬難 加入 警隊 加入 警隊 的 初心 是 止 暴 制 亂</td>\n",
              "      <td>歲 的 陳 悅 晉 剛 於 警察 學院 畢業 與 妻子 育 一 子 一 女 投身 警隊 前 ...</td>\n",
              "      <td>年 黑 暴 肆虐 暴徒 四處 破壞 摧毀 其 安穩 的 生活 年初 他 因公 司 經營 困難...</td>\n",
              "      <td>568</td>\n",
              "      <td>https://lihkg.com/thread/3411216/page/1</td>\n",
              "      <td>5.0</td>\n",
              "      <td>時事台</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>680</th>\n",
              "      <td>101688</td>\n",
              "      <td>弘 豬 天 一 食</td>\n",
              "      <td>韓國 野 或 日本 野 人 想見 著 黑 衫 牛仔褲</td>\n",
              "      <td>變 左 沐 八 既 人 約 左 巴打 天水圍 飽 屙 尋晚 唸 住 一個 韓燒 去到 尖咀 ...</td>\n",
              "      <td>96</td>\n",
              "      <td>https://lihkg.com/thread/101688/page/1</td>\n",
              "      <td>16.0</td>\n",
              "      <td>飲食台</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>57</th>\n",
              "      <td>100128</td>\n",
              "      <td>推 完場 曼聯 對 韋 根</td>\n",
              "      <td>NaN</td>\n",
              "      <td>乜事 無 曬 人 靚 月巴</td>\n",
              "      <td>6</td>\n",
              "      <td>https://lihkg.com/thread/100128/page/1</td>\n",
              "      <td>6.0</td>\n",
              "      <td>體育台</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2757</th>\n",
              "      <td>800010</td>\n",
              "      <td>同一 齊 學 貓 叫</td>\n",
              "      <td>一齊 喵 喵 喵 喵 喵</td>\n",
              "      <td>愛心 和 那個 了 雙馬尾 主要 教人 養 貓 狗 幫 聯絡 下 樓主 射精 了 隻 貓 舐...</td>\n",
              "      <td>46</td>\n",
              "      <td>https://lihkg.com/thread/800010/page/1</td>\n",
              "      <td>7.0</td>\n",
              "      <td>娛樂台</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1398</th>\n",
              "      <td>103447</td>\n",
              "      <td>後 回憶 唱 唱咩歌</td>\n",
              "      <td>唱 好多 歌 活 著 失戀 王 好心 報 最後 祝福 無賴 開始 戀愛 好多 識 唱 宜家 ...</td>\n",
              "      <td>最後 祝福 首 有得 唱 好似 套 劇 出 左 冇耐 最後 祝福 首 有得 唱 好似 套 劇...</td>\n",
              "      <td>159</td>\n",
              "      <td>https://lihkg.com/thread/103447/page/1</td>\n",
              "      <td>7.0</td>\n",
              "      <td>娛樂台</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5052</th>\n",
              "      <td>3579611</td>\n",
              "      <td>究竟 請 請</td>\n",
              "      <td>見工 時 好似 輕鬆 又帶 參觀 下個 以為 有機 點知 左 個半 禮拜 消息</td>\n",
              "      <td>移民 細 公司 見 老闆 當住 請 努力 搵 啦 見 完工 不嬲 諗住 見 見工 時 好似 ...</td>\n",
              "      <td>38</td>\n",
              "      <td>https://lihkg.com/thread/3579611/page/1</td>\n",
              "      <td>14.0</td>\n",
              "      <td>上班台</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3539</th>\n",
              "      <td>1001821</td>\n",
              "      <td>唔知 蔥蔥 開心</td>\n",
              "      <td>憂心 蔥蔥 魷魚 章魚 墨魚 三 打工 中途 休息 先 魷魚 魷 落場</td>\n",
              "      <td>魷 樂 場 好笑 係唔係 開 對 衝擊 太大 先 呢啲 唉 唔好 提 喇 先 開 如 如 如...</td>\n",
              "      <td>55</td>\n",
              "      <td>https://lihkg.com/thread/1001821/page/1</td>\n",
              "      <td>31.0</td>\n",
              "      <td>創意台</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1681</th>\n",
              "      <td>104126</td>\n",
              "      <td>寫 寫 死</td>\n",
              "      <td>聽日 仲未 寫 就係 一睇 明 題目 偏門 搵 類似 字 寫 左 野 寫 係咪 玩完 收皮</td>\n",
              "      <td>係唔係 寫 漏 聽日 仲未 寫 就係 一睇 明 題目 偏門 搵 類似 字 寫 左 野 寫 係...</td>\n",
              "      <td>93</td>\n",
              "      <td>https://lihkg.com/thread/104126/page/1</td>\n",
              "      <td>18.0</td>\n",
              "      <td>學術台</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>182</th>\n",
              "      <td>100483</td>\n",
              "      <td>求 鄭中基 今晚 單 頭 飛</td>\n",
              "      <td>NaN</td>\n",
              "      <td>一個人 演唱會 單張 幫 幫手 俾 忍心 見到 一個人 戇鳩鳩 其實 真心 粉絲 今日 打算...</td>\n",
              "      <td>100</td>\n",
              "      <td>https://lihkg.com/thread/100483/page/1</td>\n",
              "      <td>7.0</td>\n",
              "      <td>娛樂台</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6ff5bac5-8b54-4ad2-82f9-2a127c64314b')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-6ff5bac5-8b54-4ad2-82f9-2a127c64314b button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-6ff5bac5-8b54-4ad2-82f9-2a127c64314b');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-7d7802c8-f80c-4df5-91e4-255aa1dc7661\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-7d7802c8-f80c-4df5-91e4-255aa1dc7661')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-7d7802c8-f80c-4df5-91e4-255aa1dc7661 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"df\",\n  \"rows\": 10,\n  \"fields\": [\n    {\n      \"column\": \"post_number\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          \"104126\",\n          \"3411216\",\n          \"103447\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"title\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          \"\\u5beb \\u5beb \\u6b7b\",\n          \"\\u6b72 \\u4e2d\\u5e74 \\u7236\\u89aa \\u9673 \\u6085 \\u6649 \\u6392\\u9664\\u842c\\u96e3 \\u52a0\\u5165 \\u8b66\\u968a \\u52a0\\u5165 \\u8b66\\u968a \\u7684 \\u521d\\u5fc3 \\u662f \\u6b62 \\u66b4 \\u5236 \\u4e82\",\n          \"\\u5f8c \\u56de\\u61b6 \\u5531 \\u5531\\u54a9\\u6b4c\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"main_content\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 8,\n        \"samples\": [\n          \"\\u6b72 \\u7684 \\u9673 \\u6085 \\u6649 \\u525b \\u65bc \\u8b66\\u5bdf \\u5b78\\u9662 \\u7562\\u696d \\u8207 \\u59bb\\u5b50 \\u80b2 \\u4e00 \\u5b50 \\u4e00 \\u5973 \\u6295\\u8eab \\u8b66\\u968a \\u524d \\u5f9e\\u4e8b \\u73bb\\u7483 \\u5b89\\u88dd \\u5de5\\u7a0b \\u903e \\u5e74 \\u5df2 \\u6649\\u5347 \\u81f3 \\u4e3b\\u7ba1 \\u7d1a\\u5225 \\u81f3 \\u5e74 \\u9ed1 \\u66b4 \\u8086\\u8650 \\u52a0\\u4e0a \\u5e74\\u521d \\u4ed6 \\u56e0\\u516c \\u53f8 \\u7d93\\u71df \\u56f0\\u96e3 \\u88ab \\u88c1\\u54e1 \\u4ee4 \\u4ed6 \\u840c\\u751f \\u52a0\\u5165 \\u8b66\\u968a \\u60f3\\u6cd5 \\u634d\\u885b \\u6cd5\\u6cbb \\u7576\\u6642 \\u773c\\u898b \\u9752\\u5c11\\u5e74 \\u88ab \\u8aa4\\u5c0e \\u6295\\u8eab \\u9ed1 \\u66b4 \\u4ee4 \\u5fc3\\u4e2d \\u71c3\\u8d77 \\u4e00\\u5718\\u706b \\u5e0c\\u671b \\u633a\\u8eab\\u800c\\u51fa \\u5236\\u6b62 \\u66b4\\u529b \\u4fdd\\u8b77 \\u9752\\u5c11\\u5e74\",\n          \"\\u898b\\u5de5 \\u6642 \\u597d\\u4f3c \\u8f15\\u9b06 \\u53c8\\u5e36 \\u53c3\\u89c0 \\u4e0b\\u500b \\u4ee5\\u70ba \\u6709\\u6a5f \\u9ede\\u77e5 \\u5de6 \\u500b\\u534a \\u79ae\\u62dc \\u6d88\\u606f\",\n          \"\\u71d2 \\u5c71 \\u4fc2\\u54aa \\u653e \\u5de6 \\u91dd \\u4ee5\\u8ecd \\u53c3\\u8b00\\u9577 \\u856d \\u82e5 \\u5143 \\u4f9d\\u5bb6 \\u597d\\u591a \\u8ecd\\u4e8b \\u4eba\\u4eba \\u4ef2 \\u767c \\u7dca \\u5922 \\u6839\\u672c \\u5514\\u77e5 \\u9053 \\u5834 \\u4ed7 \\u9ede\\u6253 \\u6cd5 \\u4f9d\\u5bb6 \\u4ee5\\u8272\\u5217 \\u8fa6\\u6cd5 \\u5c31\\u4fc2 \\u597d\\u4f3c \\u75c5\\u4eba \\u53cd\\u8f49 \\u8eab \\u63db \\u5e8a \\u55ae \\u4f9d\\u5bb6 \\u4ee5\\u8272\\u5217 \\u5c31\\u4fc2 \\u5c07 \\u52a0\\u6c99 \\u5317\\u90e8 \\u4eba \\u9077\\u79fb \\u5357\\u908a \\u54c8\\u99ac\\u65af \\u6703\\u6b7b \\u5b88 \\u4ee5\\u8272\\u5217 \\u5c31\\u53ef\\u4ee5 \\u6e05\\u527f \\u5317\\u90e8 \\u5269\\u8fd4 \\u4eba \\u6e05\\u527f \\u54c2 \\u5730\\u9053 \\u4eba \\u6349 \\u6652 \\u51fa \\u569f \\u9010\\u500b \\u62f7\\u554f \\u8d77 \\u6652 \\u6652 \\u6240\\u6709 \\u6b66\\u5668 \\u51fa \\u569f \\u5c31\\u53ef\\u4ee5 \\u53eb \\u5357\\u90e8 \\u4eba \\u5317\\u90e8 \\u4e2d\\u9014 \\u6bcf\\u500b\\u4eba \\u641c \\u904e\\u8eab \\u5a66\\u5b7a \\u6b66\\u5668 \\u653e \\u904e\\u53bb \\u6eff\\u6eff \\u9b0d\\u9b1a \\u6574\\u500b \\u6559\\u80b2 \\u71df \\u56f0\\u4f4f \\u5148\\u7747 \\u5413 \\u54c8\\u99ac\\u65af \\u5b9a\\u4fc2 \\u5e73\\u6c11 \\u5605\\u8a71 \\u6349 \\u5be9\\u8a0a \\u7576 \\u4eba \\u4e0a \\u8fd4\\u53bb \\u5317\\u90e8 \\u4ee5\\u8272\\u5217 \\u6e05 \\u5357\\u908a \\u5c07 \\u5357\\u908a \\u4e00 \\u69d8 \\u6d17 \\u4e00\\u6b21 \\u6210\\u500b \\u5730\\u5340 \\u6e05\\u9664 \\u6652 \\u7d55\\u5c0d \\u9694\\u7d55 \\u6652 \\u5206\\u8fa8 \\u6652 \\u52a0\\u6c99 \\u5883\\u5167 \\u54c8\\u99ac\\u65af \\u5206\\u5b50 \\u62d8\\u6355 \\u6652 \\u6d3b\\u8e8d \\u5206\\u5b50\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"comments\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          \"\\u4fc2\\u5514\\u4fc2 \\u5beb \\u6f0f \\u807d\\u65e5 \\u4ef2\\u672a \\u5beb \\u5c31\\u4fc2 \\u4e00\\u7747 \\u660e \\u984c\\u76ee \\u504f\\u9580 \\u6435 \\u985e\\u4f3c \\u5b57 \\u5beb \\u5de6 \\u91ce \\u5beb \\u4fc2\\u54aa \\u73a9\\u5b8c \\u6536\\u76ae \\u65e2 \\u60c5\\u6cc1 \\u4e00\\u6a23 \\u5206\\u5206\\u9418 \\u4ef2 \\u5beb \\u5c11 \\u4f46 \\u6700\\u5f8c \\u7167 \\u4ea4 \\u807d\\u65e5 \\u4ef2\\u672a \\u5beb \\u5c31\\u4fc2 \\u4e00\\u7747 \\u660e \\u984c\\u76ee \\u504f\\u9580 \\u6435 \\u985e\\u4f3c \\u5b57 \\u5beb \\u5de6 \\u91ce \\u5beb \\u4fc2\\u54aa \\u73a9\\u5b8c \\u6536\\u76ae \\u65e2 \\u60c5\\u6cc1 \\u4e00\\u6a23 \\u5206\\u5206\\u9418 \\u4ef2 \\u5beb \\u5c11 \\u4f46 \\u6700\\u5f8c \\u7167 \\u4ea4 \\u52c1 \\u5566 \\u4f9d\\u5bb6 \\u4ef2 \\u5beb \\u7dca \\u901a\\u9802 \\u641e\\u5230 \\u5c31\\u5feb \\u5614 \\u807d\\u65e5 \\u4ef2\\u672a \\u5beb \\u5c31\\u4fc2 \\u4e00\\u7747 \\u660e \\u984c\\u76ee \\u504f\\u9580 \\u6435 \\u985e\\u4f3c \\u5b57 \\u5beb \\u5de6 \\u91ce \\u5beb \\u4fc2\\u54aa \\u73a9\\u5b8c \\u6536\\u76ae \\u8a66\\u4e0b \\u6435 \\u4e0b \\u904e\\u53bb\",\n          \"\\u5e74 \\u9ed1 \\u66b4 \\u8086\\u8650 \\u66b4\\u5f92 \\u56db\\u8655 \\u7834\\u58de \\u6467\\u6bc0 \\u5176 \\u5b89\\u7a69 \\u7684 \\u751f\\u6d3b \\u5e74\\u521d \\u4ed6 \\u56e0\\u516c \\u53f8 \\u7d93\\u71df \\u56f0\\u96e3 \\u88ab \\u88c1\\u54e1 \\u6709\\u5fc3 \\u5e74 \\u5373\\u523b \\u6295\\u8003 \\u5e74 \\u88c1\\u54e1 \\u5148\\u8003 \\u70ba \\u7ce7 \\u5566 \\u5e74\\u521d \\u4ed6 \\u56e0\\u516c \\u53f8 \\u7d93\\u71df \\u56f0\\u96e3 \\u88ab \\u88c1\\u54e1 \\u5e74\\u521d \\u4ed6 \\u56e0\\u516c \\u53f8 \\u7d93\\u71df \\u56f0\\u96e3 \\u88ab \\u88c1\\u54e1 \\u5e74\\u521d \\u4ed6 \\u56e0\\u516c \\u53f8 \\u7d93\\u71df \\u56f0\\u96e3 \\u88ab \\u88c1\\u54e1 \\u6bd4 \\u4eba \\u7092 \\u8f49\\u5de5 \\u6709\\u54a9 \\u4f9d\\u5bb6 \\u4e2d\\u5e74 \\u4ee5\\u4e0a \\u7576\\u5dee \\u6279 \\u7406 \\u6cbb\\u5b89 \\u5de5\\u4f5c \\u975e\\u5e38 \\u76ee\\u6a19 \\u5c0e\\u5411 \\u985e \\u4eba \\u5e73\\u65e5 \\u5c0f\\u4e8b \\u7169 \\u53ea \\u5e73\\u4e82 \\u7acb\\u529f \\u8b66\\u5bdf \\u826f\\u5fc3 \\u4f01\\u696d \\u5c08 \\u6536 \\u5783\\u573e \\u7576\\u6642 \\u773c\\u898b \\u9752\\u5c11\\u5e74 \\u88ab \\u8aa4\\u5c0e \\u6295\\u8eab \\u9ed1 \\u66b4 \\u4ee4 \\u5fc3\\u4e2d \\u71c3\\u8d77 \\u4e00\\u5718\\u706b \\u5e0c\\u671b \\u633a\\u8eab\\u800c\\u51fa \\u5236\\u6b62 \\u66b4\\u529b \\u4fdd\\u8b77 \\u9752\\u5c11\\u5e74 \\u6253\\u9ce9 \\u73ed \\u5f8c\\u751f\\u4ed4 \\u4e86 \\u4ffe \\u4eba \\u7092 \\u5de6 \\u91ce \\u6488 \\u4e2d\\u5e74 \\u5371\\u6a5f \\u662f\\u4f46 \\u6435 \\u6613 \\u8acb \\u65e2 \\u5de5 \\u9ece \\u505a \\u721b \\u73bb\\u7483 \\u908a\\u6709 \\u5481\\u591a \\u5730 \\u88dd \\u9673 \\u6085 \\u6649 \\u525b \\u5728 \\u8b66\\u5bdf \\u5b78\\u9662 \\u7562\\u696d \\u7372 \\u6d3e\\u99d0 \\u9ec3\\u5927\\u4ed9 \\u8b66\\u5340 \\u8ecd\\u88dd \\u5de1\\u908f \\u5c0f\\u968a \\u4ed6 \\u5728 \\u8b66 \\u8072 \\u8a2a\\u554f \\u4e2d \\u8868\\u793a \\u6295\\u8eab \\u8b66\\u968a \\u524d \\u5f9e\\u4e8b \\u73bb\\u7483 \\u5b89\\u88dd \\u5de5\\u7a0b \\u903e \\u5e74 \\u5df2 \\u6649\\u5347 \\u81f3 \\u4e3b\\u7ba1 \\u7d1a\\u5225 \\u81f3 \\u5e74 \\u9ed1 \\u66b4 \\u8086\\u8650 \\u66b4\\u5f92 \\u56db\\u8655 \\u7834\\u58de \\u6467\\u6bc0 \\u5176 \\u5b89\\u7a69 \\u7684 \\u751f\\u6d3b \\u5e74\\u521d \\u4ed6 \\u56e0\\u516c \\u53f8 \\u7d93\\u71df \\u56f0\\u96e3 \\u88ab \\u88c1\\u54e1 \\u6545 \\u6c7a\\u5fc3 \\u52a0\\u5165 \\u8b66\\u968a \\u5e74 \\u5c46 \\u4e2d\\u5e74 \\u7684 \\u9673 \\u6085 \\u6649 \\u52a0\\u5165 \\u8b66\\u968a \\u7684 \\u8def\\u9014 \\u4e26\\u4e0d \\u5e73\\u5766 \\u4ed6 \\u5411 \\u8b66 \\u8072 \\u900f\\u9732 \\u9664\\u4e86 \\u5b78\\u6b77 \\u53ca \\u9ad4\\u80fd \\u65b9\\u9762 \\u9700 \\u9762\\u5c0d \\u773e \\u591a\\u5e74 \\u8f15 \\u7af6\\u722d\\u8005 \\u592a\\u592a \\u4ea6 \\u64d4\\u5fc3 \\u5b50\\u5973 \\u56e0\\u6b64 \\u53d7 \\u6b3a\\u51cc \\u53cd\\u5c0d \\u4ed6 \\u6295\\u8003 \\u8b66\\u968a \\u5169 \\u4eba \\u66f4 \\u70ba \\u6b64 \\u756a \\u722d\\u57f7 \\u70ba \\u9054\\u6210 \\u76ee\\u6a19 \\u4ed6 \\u9010\\u4e00 \\u514b\\u670d \\u56f0\\u96e3 \\u5148 \\u627e \\u517c\\u8077 \\u5de5\\u4f5c \\u89e3\\u6c7a \\u751f\\u8a08 \\u554f\\u984c \\u65bc \\u5de5\\u9918 \\u6642\\u9593 \\u9032\\u4fee \\u4e26 \\u7d93\\u5e38 \\u62bd\\u7a7a \\u8dd1\\u6b65 \\u935b\\u934a \\u9ad4\\u80fd \\u4ee5 \\u9054\\u5230 \\u5165 \\u8077 \\u8981\\u6c42 \\u9673 \\u6085 \\u6649 \\u7684 \\u592a\\u592a \\u6700\\u7d42 \\u88ab \\u9673 \\u6085 \\u6649 \\u7684 \\u5805\\u6bc5 \\u6240 \\u6253\\u52d5 \\u652f\\u6301 \\u4ed6 \\u8ffd\\u5c0b \\u8b66\\u5bdf \\u5922 \\u81f3 \\u53bb\\u5e74 \\u7d42 \\u7372\\u53d6 \\u9304 \\u63a5\\u53d7 \\u70ba \\u671f \\u5468 \\u7684 \\u5b78 \\u8b66 \\u57fa\\u790e \\u8a13\\u7df4 \\u8ab2\\u7a0b \\u9673 \\u6085 \\u6649 \\u4ecd \\u8981\\u9762 \\u5c0d \\u4e0d\\u5c11 \\u6311\\u6230 \\u7279\\u5225 \\u662f \\u4eba \\u4e2d\\u5e74 \\u91cd\\u62fe \\u66f8\\u672c \\u4e26\\u975e \\u6613 \\u4e8b \\u4ed6 \\u5766 \\u8a00 \\u7576\\u6642 \\u8003\\u8a66 \\u6210\\u7e3e \\u5982 \\u7406\\u60f3 \\u975e\\u5e38 \\u64d4\\u5fc3 \\u7121\\u6cd5 \\u7562\\u696d \\u9673 \\u6085 \\u6649 \\u5341\\u5206 \\u611f\\u8b1d \\u7576\\u6642 \\u7684 \\u73ed\\u4e3b\\u4efb \\u5c0d\\u65b9 \\u9664 \\u7d66\\u4e88 \\u9f13\\u52f5 \\u548c \\u5206\\u4eab \\u8b80\\u66f8 \\u5fc3\\u5f97 \\u66f4 \\u5b89\\u6392 \\u5b78\\u8853 \\u8868\\u73fe \\u8f03\\u4f73 \\u7684 \\u540c\\u5b78 \\u665a\\u4e0a \\u70ba \\u4ed6 \\u88dc\\u7fd2 \\u6700\\u7d42 \\u4ed6 \\u9806\\u5229 \\u53d6 \\u7406\\u60f3 \\u6210\\u7e3e \\u5982\\u9858 \\u52a0\\u5165 \\u8b66\\u968a \\u9673 \\u6085 \\u6649 \\u8868\\u793a \\u52a0\\u5165 \\u8b66\\u968a \\u7684 \\u521d\\u5fc3 \\u662f \\u6b62 \\u66b4 \\u5236 \\u4e82 \\u96d6\\u7136 \\u76ee\\u524d \\u793e\\u6703\\u74b0\\u5883 \\u76f8\\u5c0d \\u7a69\\u5b9a \\u4f46 \\u4ed6 \\u7d55\\u5c0d \\u4e0d\\u6703 \\u6389\\u4ee5\\u8f15\\u5fc3 \\u5805\\u5b88 \\u4fdd\\u8b77 \\u9999\\u6e2f \\u7684 \\u524d\\u7dda \\u9632\\u7bc4 \\u540c\\u985e \\u4e8b\\u4ef6 \\u518d\\u6b21 \\u767c\\u751f \\u4ed6 \\u4ea6 \\u5e0c\\u671b \\u80fd \\u6210\\u70ba \\u5b50\\u5973 \\u7684 \\u699c\\u6a23 \\u5b78 \\u61c2 \\u6392\\u9664\\u842c\\u96e3 \\u70ba \\u76ee\\u6a19 \\u8fce \\u96e3 \\u4e0a \\u6bcf\\u65e5 \\u5132 \\u868a \\u4e09\\u5e74 \\u5f8c \\u7528 \\u81ea\\u8eab \\u7a4d\\u84c4 \\u52a0\\u57cb \\u7238\\u7238 \\u6bd4 \\u65e2 \\u842c \\u4e0a \\u6a13 \\u5165 \\u592a\\u592a \\u4ea6 \\u64d4\\u5fc3 \\u5b50\\u5973 \\u56e0\\u6b64 \\u53d7 \\u6b3a\\u51cc \\u53cd\\u5c0d \\u4ed6 \\u6295\\u8003 \\u8b66\\u968a \\u5168\\u5bb6 \\u72d7 \\u649a \\u52f5\\u5fd7 \\u52a0\\u4e0a \\u5e74\\u521d \\u4ed6 \\u56e0\\u516c \\u53f8 \\u7d93\\u71df \\u56f0\\u96e3 \\u88ab \\u88c1\\u54e1 \\u505a \\u5730\\u76e4 \\u505a\\u5230 \\u4ffe \\u4eba \\u96de \\u51fa \\u569f \\u9072 \\u5e74\\u6b72 \\u7684 \\u9673 \\u6085 \\u6649 \\u525b \\u65bc \\u8b66\\u5bdf \\u5b78\\u9662 \\u7562\\u696d \\u8207 \\u59bb\\u5b50 \\u80b2 \\u4e00 \\u5b50 \\u4e00 \\u5973 \\u6295\\u8eab \\u8b66\\u968a \\u524d \\u5f9e\\u4e8b \\u73bb\\u7483 \\u5b89\\u88dd \\u5de5\\u7a0b \\u903e \\u5e74 \\u5df2 \\u6649\\u5347 \\u81f3 \\u4e3b\\u7ba1 \\u7d1a\\u5225 \\u81f3 \\u5e74 \\u9ed1 \\u66b4 \\u8086\\u8650 \\u52a0\\u4e0a \\u5e74\\u521d \\u4ed6 \\u56e0\\u516c \\u53f8 \\u7d93\\u71df \\u56f0\\u96e3 \\u88ab \\u88c1\\u54e1 \\u4ee4 \\u4ed6 \\u840c\\u751f \\u52a0\\u5165 \\u8b66\\u968a \\u60f3\\u6cd5 \\u634d\\u885b \\u6cd5\\u6cbb \\u7576\\u6642 \\u773c\\u898b \\u9752\\u5c11\\u5e74 \\u88ab \\u8aa4\\u5c0e \\u6295\\u8eab \\u9ed1 \\u66b4 \\u4ee4 \\u5fc3\\u4e2d \\u71c3\\u8d77 \\u4e00\\u5718\\u706b \\u5e0c\\u671b \\u633a\\u8eab\\u800c\\u51fa \\u5236\\u6b62 \\u66b4\\u529b \\u4fdd\\u8b77 \\u9752\\u5c11\\u5e74 \\u592a\\u592a \\u4ea6 \\u64d4\\u5fc3 \\u5b50\\u5973 \\u56e0\\u6b64 \\u53d7 \\u6b3a\\u51cc \\u8e29 \\u9ce9 \\u4efd\\u5de5 \\u556b \\u6bc5\\u9032 \\u7562\\u696d \\u53e4\\u8a9e\\u6709\\u4e91 \\u53c8\\u8981 \\u505a\\u96de \\u53c8\\u8981 \\u8c9e\\u7bc0 \\u6392 \\u574a \\u5e7e\\u6642 \\u7747\\u4f4f \\u540c\\u4e8b \\u4e00 \\u96bb\\u96bb \\u72af\\u7f6a \\u611f\\u89ba \\u4ef2\\u6709 \\u5e74 \\u597d\\u591a \\u88ab\\u81ea\\u6bba \\u6848 \\u904e\\u773c \\u4ef2\\u6709 \\u66b4\",\n          \"\\u6700\\u5f8c \\u795d\\u798f \\u9996 \\u6709\\u5f97 \\u5531 \\u597d\\u4f3c \\u5957 \\u5287 \\u51fa \\u5de6 \\u5187\\u8010 \\u6700\\u5f8c \\u795d\\u798f \\u9996 \\u6709\\u5f97 \\u5531 \\u597d\\u4f3c \\u5957 \\u5287 \\u51fa \\u5de6 \\u5187\\u8010 \\u6709\\u7121 \\u5531 \\u9673 \\u5ee3 \\u5049 \\u52c1\\u6b4c\\u91d1\\u66f2 \\u91cd\\u53e3\\u5473 \\u6700\\u5f8c \\u795d\\u798f \\u9996 \\u6709\\u5f97 \\u5531 \\u4f9d\\u5bb6 \\u4ef2\\u6709 \\u4eba \\u5531 \\u5566 \\u5531 \\u5a01 \\u4f9d\\u5bb6 \\u5f8c\\u751f \\u7576 \\u5462\\u5572 \\u8001\\u9905 \\u6b4c \\u4e86 \\u5531 \\u8b5a\\u8a60\\u9e9f \\u5bb9\\u7956\\u5152 \\u820a\\u6b4c \\u7121 \\u8f38 \\u4ef2\\u6709 \\u5152\\u6b4c \\u8d85\\u7d1a \\u5c0f \\u9ed1 \\u4fdd \\u5927 \\u7f8e \\u5c11\\u5973 \\u6230\\u58eb \\u5c0f \\u4e38\\u5b50 \\u5c39\\u5149 \\u7238\\u7238 \\u5c11\\u7406 \\u963f\\u7238 \\u6b63 \\u4ec6\\u8857 \\u4f9d\\u5bb6 \\u4ef2\\u6709 \\u4eba \\u5531 \\u5566 \\u5531 \\u5a01 \\u4e8b\\u5be6 \\u6562 \\u807d \\u4f9d\\u5bb6 \\u5f8c\\u751f \\u7576 \\u5462\\u5572 \\u8001\\u9905 \\u6b4c \\u4e86 \\u5f8c \\u5514\\u6703 \\u8b58 \\u83ef \\u661f \\u82f1\\u7687 \\u5e74\\u4ee3 \\u820a\\u6b4c \\u5187\\u5f97 \\u8f38 \\u5764 \\u54e5 \\u5f8c \\u56e1\\u56e1 \\u6253 \\u95dc\\u4fc2 \\u6cb9 \\u4e0d\\u662f \\u516b\\u5341\\u5e74\\u4ee3 \\u4f9d\\u5bb6 \\u4ef2\\u6709 \\u4eba \\u5531 \\u5764 \\u54e5 \\u5f8c \\u56e1\\u56e1 \\u6253 \\u95dc\\u4fc2 \\u5c39\\u5149 \\u5531 \\u597d\\u591a \\u6b4c \\u6d3b \\u8457 \\u5931\\u6200 \\u738b \\u597d\\u5fc3 \\u5831 \\u6700\\u5f8c \\u795d\\u798f \\u7121\\u8cf4 \\u958b\\u59cb \\u6200\\u611b \\u597d\\u591a \\u8b58 \\u5531 \\u5b9c\\u5bb6 \\u5187\\u4eba \\u5531 \\u4f9d\\u5bb6 \\u5f8c\\u751f \\u7576 \\u5462\\u5572 \\u8001\\u9905 \\u6b4c \\u4e86 \\u5f8c \\u5514\\u6703 \\u8b58 \\u59d0 \\u649a \\u5ee3 \\u5049 \\u6cb9 \\u4e0d\\u662f \\u516b\\u5341\\u5e74\\u4ee3 \\u98f2\\u6b4c \\u4ef2\\u8981 \\u8df3\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"word_count\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          \"93\",\n          \"568\",\n          \"159\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"URL\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          \"https://lihkg.com/thread/104126/page/1\",\n          \"https://lihkg.com/thread/3411216/page/1\",\n          \"https://lihkg.com/thread/103447/page/1\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Category\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 7,\n        \"samples\": [\n          \"5.0\",\n          \"16.0\",\n          \"31.0\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Classification\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 7,\n        \"samples\": [\n          \"\\u6642\\u4e8b\\u53f0\",\n          \"\\u98f2\\u98df\\u53f0\",\n          \"\\u5275\\u610f\\u53f0\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 9
        }
      ],
      "source": [
        "import pandas as pd\n",
        "\n",
        "# Load the dataset into a pandas dataframe.\n",
        "df = pd.read_csv(\"/content/preprocessed_data(210).csv\", names=['post_number','title','main_content','comments', 'word_count', 'URL', 'Category', 'Classification'])\n",
        "\n",
        "# Report the number of sentences.\n",
        "print('Number of training sentences: {:,}\\n'.format(df.shape[0]))\n",
        "\n",
        "# Display 10 random rows from the data.\n",
        "df.sample(10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "PbTb7wlBxG5o",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0c1892a5-44d2-431b-bf52-a2ea59b7201e"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "7204"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ],
      "source": [
        "len(df)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "yAUdAC1WxOgw",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "215b627f-c5b8-410b-86b1-cdb5c516a126"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "6090"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ],
      "source": [
        "df = df.dropna(axis=0, how='any')\n",
        "len(df)"
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "EepJSWPgBJQ5"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "possible_labels = df.Category[1:].unique()\n",
        "label_dict = {}\n",
        "for index, possible_label in enumerate(possible_labels):\n",
        "    label_dict[possible_label] = index\n",
        "label_dict"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vbAP1ut5BNqC",
        "outputId": "3e998921-ee7f-4876-f225-1c13b7d34d32"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'28.0': 0,\n",
              " '5.0': 1,\n",
              " '33.0': 2,\n",
              " '31.0': 3,\n",
              " '30.0': 4,\n",
              " '21.0': 5,\n",
              " '8.0': 6,\n",
              " '16.0': 7,\n",
              " '6.0': 8,\n",
              " '20.0': 9,\n",
              " '22.0': 10,\n",
              " '23.0': 11,\n",
              " '18.0': 12,\n",
              " '10.0': 13,\n",
              " '17.0': 14,\n",
              " '25.0': 15,\n",
              " '11.0': 16,\n",
              " '13.0': 17,\n",
              " '4.0': 18,\n",
              " '7.0': 19,\n",
              " '14.0': 20,\n",
              " '24.0': 21,\n",
              " '34.0': 22,\n",
              " '32.0': 23,\n",
              " '12.0': 24,\n",
              " '9.0': 25,\n",
              " '26.0': 26,\n",
              " '19.0': 27,\n",
              " '15.0': 28,\n",
              " '27.0': 29,\n",
              " '35.0': 30,\n",
              " '36.0': 31,\n",
              " '37.0': 32,\n",
              " '38.0': 33,\n",
              " '39.0': 34,\n",
              " '40.0': 35,\n",
              " '41.0': 36}"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "RKTPIxeyBNsI"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "_JOY18QSo-a0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3e161ac2-d0b4-41cd-a92e-bfee0adef8ac"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(6089,)\n",
            "(6089,)\n"
          ]
        }
      ],
      "source": [
        "sentences = df.comments[1:].values\n",
        "labels = df.Category[1:].values\n",
        "print(labels.shape)\n",
        "print(sentences.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "qGpWgGSTpRsI",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b7b7df5b-b4c6-4646-891f-7061e74df62a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading BERT tokenizer...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "from transformers import BertTokenizer\n",
        "\n",
        "# Load the BERT tokenizer.\n",
        "print('Loading BERT tokenizer...')\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "ie9OMZswphHp",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7ee666f7-dbd3-423f-ce30-681b8b24a82d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " Original:  京津 照 放炮 仗 污染 爆錶 另一方面 今年 春節 假期 留 在 國內 的 民 眾 則 繼續 忍受 霧霾 之 苦 包括 北京 等 國內 城市 近日 均 對 市民 燃放 煙花 爆竹 祭 出 嚴 控 姿態 惟 民 眾 仍 堅持 爆竹 一聲 除 舊 歲 其中 京津 冀 大批 民 眾 前晚 開始 燃放 炮仗 令 區域 懸浮粒子 濃度 急 升官 方 公佈 北京 的 空氣 質量 指數 在 昨 凌晨 時 至 時 超過 達 嚴重 污染 的 爆錶 水平 若 用 美國 標準 更 曾 升 至 的 峯值 至 昨早 天明 時 才 有所改善 京津 照 放炮 仗 污染 爆錶 另一方面 今年 春節 假期 留 在 國內 的 民 眾 則 繼續 忍受 霧霾 之 苦 包括 北京 等 國內 城市 近日 均 對 市民 燃放 煙花 爆竹 祭 出 嚴 控 姿態 惟 民 眾 仍 堅持 爆竹 一聲 除 舊 歲 其中 京津 冀 大批 民 眾 前晚 開始 燃放 炮仗 令 區域 懸浮粒子 濃度 急 升官 方 公佈 北京 的 空氣 質量 指數 在 昨 凌晨 時 至 時 超過 達 嚴重 污染 的 爆錶 水平 若 用 美國 標準 更 曾 升 至 的 峯值 至 昨早 天明 時 才 有所改善 支那 另外 有一套 標準 香港 行緊 另一 套 標準 支那 點會 放假 毒 霧 吸 慣 京津 照 放炮 仗 污染 爆錶 另一方面 今年 春節 假期 留 在 國內 的 民 眾 則 繼續 忍受 霧霾 之 苦 包括 北京 等 國內 城市 近日 均 對 市民 燃放 煙花 爆竹 祭 出 嚴 控 姿態 惟 民 眾 仍 堅持 爆竹 一聲 除 舊 歲 其中 京津 冀 大批 民 眾 前晚 開始 燃放 炮仗 令 區域 懸浮粒子 濃度 急 升官 方 公佈 北京 的 空氣 質量 指數 在 昨 凌晨 時 至 時 超過 達 嚴重 污染 的 爆錶 水平 若 用 美國 標準 更 曾 升 至 的 峯值 至 昨早 天明 時 才 有所改善 支那 另外 有一套 標準\n",
            "Tokenized:  ['京', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '一', '方', '面', '[UNK]', '年', '春', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '國', '[UNK]', '的', '民', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '之', '[UNK]', '[UNK]', '[UNK]', '北', '京', '[UNK]', '國', '[UNK]', '城', '[UNK]', '[UNK]', '日', '[UNK]', '[UNK]', '[UNK]', '民', '[UNK]', '[UNK]', '[UNK]', '花', '[UNK]', '竹', '[UNK]', '出', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '民', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '竹', '一', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '中', '京', '[UNK]', '[UNK]', '大', '[UNK]', '民', '[UNK]', '前', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '子', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '方', '公', '[UNK]', '北', '京', '的', '空', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '的', '[UNK]', '[UNK]', '水', '平', '[UNK]', '[UNK]', '美', '國', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '的', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '天', '明', '[UNK]', '[UNK]', '有', '[UNK]', '[UNK]', '[UNK]', '京', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '一', '方', '面', '[UNK]', '年', '春', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '國', '[UNK]', '的', '民', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '之', '[UNK]', '[UNK]', '[UNK]', '北', '京', '[UNK]', '國', '[UNK]', '城', '[UNK]', '[UNK]', '日', '[UNK]', '[UNK]', '[UNK]', '民', '[UNK]', '[UNK]', '[UNK]', '花', '[UNK]', '竹', '[UNK]', '出', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '民', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '竹', '一', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '中', '京', '[UNK]', '[UNK]', '大', '[UNK]', '民', '[UNK]', '前', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '子', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '方', '公', '[UNK]', '北', '京', '的', '空', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '的', '[UNK]', '[UNK]', '水', '平', '[UNK]', '[UNK]', '美', '國', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '的', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '天', '明', '[UNK]', '[UNK]', '有', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '外', '有', '一', '[UNK]', '[UNK]', '[UNK]', '香', '[UNK]', '行', '[UNK]', '[UNK]', '一', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '京', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '一', '方', '面', '[UNK]', '年', '春', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '國', '[UNK]', '的', '民', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '之', '[UNK]', '[UNK]', '[UNK]', '北', '京', '[UNK]', '國', '[UNK]', '城', '[UNK]', '[UNK]', '日', '[UNK]', '[UNK]', '[UNK]', '民', '[UNK]', '[UNK]', '[UNK]', '花', '[UNK]', '竹', '[UNK]', '出', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '民', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '竹', '一', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '中', '京', '[UNK]', '[UNK]', '大', '[UNK]', '民', '[UNK]', '前', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '子', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '方', '公', '[UNK]', '北', '京', '的', '空', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '的', '[UNK]', '[UNK]', '水', '平', '[UNK]', '[UNK]', '美', '國', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '的', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '天', '明', '[UNK]', '[UNK]', '有', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '[UNK]', '外', '有', '一', '[UNK]', '[UNK]', '[UNK]']\n",
            "Token IDs:  [1755, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 1740, 1863, 1976, 100, 1840, 1867, 100, 100, 100, 100, 100, 1800, 100, 1916, 1892, 100, 100, 100, 100, 100, 100, 100, 100, 1749, 100, 100, 100, 1781, 1755, 100, 1800, 100, 1804, 100, 100, 1864, 100, 100, 100, 1892, 100, 100, 100, 1940, 100, 1933, 100, 1774, 100, 100, 100, 100, 100, 1892, 100, 100, 100, 100, 100, 1933, 1740, 100, 100, 100, 100, 100, 1746, 1755, 100, 100, 1810, 100, 1892, 100, 1776, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 1816, 100, 100, 100, 100, 100, 1863, 1772, 100, 1781, 1755, 1916, 1930, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 1916, 100, 100, 1893, 1839, 100, 100, 1935, 1800, 100, 100, 100, 100, 100, 100, 1916, 100, 100, 100, 100, 100, 1811, 1865, 100, 100, 1873, 100, 100, 100, 1755, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 1740, 1863, 1976, 100, 1840, 1867, 100, 100, 100, 100, 100, 1800, 100, 1916, 1892, 100, 100, 100, 100, 100, 100, 100, 100, 1749, 100, 100, 100, 1781, 1755, 100, 1800, 100, 1804, 100, 100, 1864, 100, 100, 100, 1892, 100, 100, 100, 1940, 100, 1933, 100, 1774, 100, 100, 100, 100, 100, 1892, 100, 100, 100, 100, 100, 1933, 1740, 100, 100, 100, 100, 100, 1746, 1755, 100, 100, 1810, 100, 1892, 100, 1776, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 1816, 100, 100, 100, 100, 100, 1863, 1772, 100, 1781, 1755, 1916, 1930, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 1916, 100, 100, 1893, 1839, 100, 100, 1935, 1800, 100, 100, 100, 100, 100, 100, 1916, 100, 100, 100, 100, 100, 1811, 1865, 100, 100, 1873, 100, 100, 100, 100, 100, 100, 1809, 1873, 1740, 100, 100, 100, 1979, 100, 1945, 100, 100, 1740, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 1755, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 1740, 1863, 1976, 100, 1840, 1867, 100, 100, 100, 100, 100, 1800, 100, 1916, 1892, 100, 100, 100, 100, 100, 100, 100, 100, 1749, 100, 100, 100, 1781, 1755, 100, 1800, 100, 1804, 100, 100, 1864, 100, 100, 100, 1892, 100, 100, 100, 1940, 100, 1933, 100, 1774, 100, 100, 100, 100, 100, 1892, 100, 100, 100, 100, 100, 1933, 1740, 100, 100, 100, 100, 100, 1746, 1755, 100, 100, 1810, 100, 1892, 100, 1776, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 1816, 100, 100, 100, 100, 100, 1863, 1772, 100, 1781, 1755, 1916, 1930, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 1916, 100, 100, 1893, 1839, 100, 100, 1935, 1800, 100, 100, 100, 100, 100, 100, 1916, 100, 100, 100, 100, 100, 1811, 1865, 100, 100, 1873, 100, 100, 100, 100, 100, 100, 1809, 1873, 1740, 100, 100, 100]\n"
          ]
        }
      ],
      "source": [
        "# Print the original sentence.\n",
        "print(' Original: ', sentences[1])\n",
        "\n",
        "# Print the sentence split into tokens.\n",
        "print('Tokenized: ', tokenizer.tokenize(sentences[1]))\n",
        "\n",
        "# Print the sentence mapped to token ids.\n",
        "print('Token IDs: ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(sentences[1])))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "UFL3CbKwt4NW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "451798a8-b131-494f-87b0-68f0511e2ae4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Token indices sequence length is longer than the specified maximum sequence length for this model (519 > 512). Running this sequence through the model will result in indexing errors\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Max sentence length:  14874\n"
          ]
        }
      ],
      "source": [
        "max_len = 0\n",
        "\n",
        "# For every sentence...\n",
        "for sent in sentences:\n",
        "\n",
        "    # Tokenize the text and add `[CLS]` and `[SEP]` tokens.\n",
        "    input_ids = tokenizer.encode(sent, add_special_tokens=True)\n",
        "\n",
        "    # Update the maximum sentence length.\n",
        "    max_len = max(max_len, len(input_ids))\n",
        "\n",
        "print('Max sentence length: ', max_len)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "5UPFtU7zLinm",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "db6f2b9e-e2c2-4502-e493-32ebc7b62428"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original:  講錢 實際 件 產品 服務 做 咪有 人 幫襯 相反 做 無咁 消失 聽到 小 商戶 做 住 事實 就係 無人 幫襯 夠 人 支持 的 消費 就係 表態 福 喜 黑心 肉 員工 食飯 無 飯 鐘 錢 全部 市民 支持 先 出現 唔好 將 責任 全部 推 俾 無良 商人 無 的 支持 點會 成功 一句 哂 新 一年 表態 消費 做 起\n",
            "Token IDs: tensor([ 101,  100,  100,  100,  100,  100,  100,  100,  100,  100,  100,  100,\n",
            "        1873, 1756,  100,  100, 1919,  100,  100,  100,  100,  100,  100,  100,\n",
            "         100, 1829,  100,  100,  100,  100, 1751,  100,  100,  100,  100, 1756,\n",
            "         100,  100,  100, 1756,  100,  100, 1916,  100,  100,  100,  100,  100,\n",
            "         100, 1926,  100,  100, 1849,  100,  100,  100, 1978,  100,  100,  100,\n",
            "         100,  100,  100,  102])\n",
            "torch.Size([6089])\n"
          ]
        }
      ],
      "source": [
        "# Tokenize all of the sentences and map the tokens to thier word IDs.\n",
        "# Convert the labels to a NumPy array\n",
        "\n",
        "labels_array = df.Category[1:].values.astype(float)\n",
        "\n",
        "# Convert the NumPy array to a tensor\n",
        "#labels = torch.tensor(labels_array)\n",
        "labels = torch.tensor(labels_array, dtype=torch.long)\n",
        "\n",
        "\n",
        "input_ids = []\n",
        "attention_masks = []\n",
        "labels_list = []\n",
        "\n",
        "# For every sentence...\n",
        "for sent in sentences:\n",
        "    # `encode_plus` will:\n",
        "    #   (1) Tokenize the sentence.\n",
        "    #   (2) Prepend the `[CLS]` token to the start.\n",
        "    #   (3) Append the `[SEP]` token to the end.\n",
        "    #   (4) Map tokens to their IDs.\n",
        "    #   (5) Pad or truncate the sentence to `max_length`\n",
        "    #   (6) Create attention masks for [PAD] tokens.\n",
        "    encoded_dict = tokenizer.encode_plus(\n",
        "                        sent,                      # Sentence to encode.\n",
        "                        add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
        "                        max_length = 64,           # Pad & truncate all sentences.\n",
        "\n",
        "                        padding='max_length',  # Use the new padding argument\n",
        "                        truncation=True,  # Explicitly activate truncation\n",
        "                        #pad_to_max_length = True,\n",
        "                        return_attention_mask = True,   # Construct attn. masks.\n",
        "                        return_tensors = 'pt',     # Return pytorch tensors.\n",
        "                   )\n",
        "\n",
        "    # Add the encoded sentence to the list.\n",
        "    input_ids.append(encoded_dict['input_ids'])\n",
        "\n",
        "    # And its attention mask (simply differentiates padding from non-padding).\n",
        "    attention_masks.append(encoded_dict['attention_mask'])\n",
        "\n",
        "    #labels_list.append(label)\n",
        "\n",
        "# Convert the lists into tensors.\n",
        "input_ids = torch.cat(input_ids, dim=0)\n",
        "attention_masks = torch.cat(attention_masks, dim=0)\n",
        "\n",
        "#labels = torch.tensor(labels)\n",
        "#labels = torch.tensor(labels, dtype=torch.long)\n",
        "# Convert the labels array to a list\n",
        "#labels = torch.cat(labels_list, dim=0)\n",
        "#labels = torch.tensor(labels_list)\n",
        "#labels = torch.tensor(labels).clone().detach()\n",
        "\n",
        "\n",
        "\n",
        "# 將 labels 的大小從 (6089,) 擴展為 (6089, 1)\n",
        "#labels = labels.unsqueeze(1)\n",
        "\n",
        "# 使用 expand 將 labels 的大小擴展為 (6089, 256)\n",
        "#labels = labels.expand(-1, 256)\n",
        "\n",
        "\n",
        "\n",
        "print('Original: ', sentences[2])\n",
        "print('Token IDs:', input_ids[2])\n",
        "print(labels.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "Mqo9PbIWOIQX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b8f0323f-0945-419f-b600-e66ff0cb48da"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([6089, 64])\n",
            "torch.Size([6089, 64])\n",
            "torch.Size([6089])\n"
          ]
        }
      ],
      "source": [
        "print(input_ids.size())\n",
        "print(attention_masks.size())\n",
        "print(labels.size())\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "id": "qXkGLQV8Q4Oo"
      },
      "outputs": [],
      "source": [
        "########## if run this step the size become （6089，） dont run！！\n",
        "\n",
        "#labels_array = df.Category[1:].values.astype(float)\n",
        "\n",
        "# Convert the NumPy array to a tensor\n",
        "#labels = torch.tensor(labels_array)\n",
        "#print(labels.size())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "bahVORyrijCT",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "25844e5c-8903-4826-80a1-14703713d446"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "5,480 training samples\n",
            "  609 validation samples\n"
          ]
        }
      ],
      "source": [
        "from torch.utils.data import TensorDataset, random_split\n",
        "\n",
        "\n",
        "#labels = df['Category'][1:].astype(float).astype(int).values\n",
        "#labels = df['Category'][1:].astype(int).values\n",
        "#labels = torch.tensor(labels).long()\n",
        "# 将标签转换为整数类型\n",
        "#labels = labels.long()\n",
        "# 将标签转换为整数类型\n",
        "#labels = torch.tensor(df['Category'][1:].values).long()\n",
        "\n",
        "# Combine the training inputs into a TensorDataset.\n",
        "dataset = TensorDataset(input_ids, attention_masks, labels)\n",
        "\n",
        "# Create a 90-10 train-validation split.\n",
        "\n",
        "# Calculate the number of samples to include in each set.\n",
        "train_size = int(0.9 * len(dataset))\n",
        "val_size = len(dataset) - train_size\n",
        "\n",
        "# Divide the dataset by randomly selecting samples.\n",
        "train_dataset, val_dataset = random_split(dataset, [train_size, val_size])\n",
        "\n",
        "print('{:>5,} training samples'.format(train_size))\n",
        "print('{:>5,} validation samples'.format(val_size))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "k53XkpGFjTfR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6f9adfe8-d620-49fc-c9b9-228ac0cc4aeb"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([ 4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21,\n",
            "        22, 23, 24, 25, 26, 27, 28, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40,\n",
            "        41])\n"
          ]
        }
      ],
      "source": [
        "import torch\n",
        "\n",
        "# 检查标签中的不同类别值\n",
        "unique_labels = torch.unique(labels)\n",
        "print(unique_labels)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "WCvrMQ78SYsP"
      },
      "outputs": [],
      "source": [
        "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
        "\n",
        "# The DataLoader needs to know our batch size for training, so we specify it\n",
        "# here. For fine-tuning BERT on a specific task, the authors recommend a batch\n",
        "# size of 16 or 32.\n",
        "batch_size = 16\n",
        "\n",
        "# Create the DataLoaders for our training and validation sets.\n",
        "# We'll take training samples in random order.\n",
        "train_dataloader = DataLoader(\n",
        "            train_dataset,  # The training samples.\n",
        "            sampler = RandomSampler(train_dataset), # Select batches randomly\n",
        "            batch_size = batch_size # Trains with this batch size.\n",
        "        )\n",
        "\n",
        "# For validation the order doesn't matter, so we'll just read them sequentially.\n",
        "validation_dataloader = DataLoader(\n",
        "            val_dataset, # The validation samples.\n",
        "            sampler = SequentialSampler(val_dataset), # Pull out batches sequentially.\n",
        "            batch_size = batch_size # Evaluate with this batch size.\n",
        "        )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "id": "RGPI2Bv_Sdhm"
      },
      "outputs": [],
      "source": [
        "### 4. Train Our Classification Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "id": "uoNfNu22VhJd",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "38ce0daf-d076-4f6b-c8ed-7439488a3bfb"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "37\n"
          ]
        }
      ],
      "source": [
        "import pandas as pd\n",
        "\n",
        "\n",
        "unique_classes = df['Category'][1:].nunique()\n",
        "\n",
        "print(unique_classes)  # 輸出不同分類的數量"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "id": "IdiEO9O-ShkJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e5966964-5ef4-4b34-9bad-abb54edc87e1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ],
      "source": [
        "from transformers import BertForSequenceClassification, AdamW, BertConfig\n",
        "\n",
        "# Load BertForSequenceClassification, the pretrained BERT model with a single\n",
        "# linear classification layer on top.\n",
        "model = BertForSequenceClassification.from_pretrained(\n",
        "    \"bert-base-uncased\", # Use the 12-layer BERT model, with an uncased vocab.\n",
        "    num_labels = 41, # The number of output labels--2 for binary classification.\n",
        "                    # You can increase this for multi-class tasks.\n",
        "    output_attentions = False, # Whether the model returns attentions weights.\n",
        "    output_hidden_states = False, # Whether the model returns all hidden-states.\n",
        ")\n",
        "\n",
        "# Tell pytorch to run this model on the GPU.\n",
        "\n",
        "#model.cuda()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "id": "xuGCiNMMWSDx",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d9abe712-cd2c-4898-c4b9-13573a71d9f4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The BERT model has 201 different named parameters.\n",
            "\n",
            "==== Embedding Layer ====\n",
            "\n",
            "bert.embeddings.word_embeddings.weight                  (30522, 768)\n",
            "bert.embeddings.position_embeddings.weight                (512, 768)\n",
            "bert.embeddings.token_type_embeddings.weight                (2, 768)\n",
            "bert.embeddings.LayerNorm.weight                              (768,)\n",
            "bert.embeddings.LayerNorm.bias                                (768,)\n",
            "\n",
            "==== First Transformer ====\n",
            "\n",
            "bert.encoder.layer.0.attention.self.query.weight          (768, 768)\n",
            "bert.encoder.layer.0.attention.self.query.bias                (768,)\n",
            "bert.encoder.layer.0.attention.self.key.weight            (768, 768)\n",
            "bert.encoder.layer.0.attention.self.key.bias                  (768,)\n",
            "bert.encoder.layer.0.attention.self.value.weight          (768, 768)\n",
            "bert.encoder.layer.0.attention.self.value.bias                (768,)\n",
            "bert.encoder.layer.0.attention.output.dense.weight        (768, 768)\n",
            "bert.encoder.layer.0.attention.output.dense.bias              (768,)\n",
            "bert.encoder.layer.0.attention.output.LayerNorm.weight        (768,)\n",
            "bert.encoder.layer.0.attention.output.LayerNorm.bias          (768,)\n",
            "bert.encoder.layer.0.intermediate.dense.weight           (3072, 768)\n",
            "bert.encoder.layer.0.intermediate.dense.bias                 (3072,)\n",
            "bert.encoder.layer.0.output.dense.weight                 (768, 3072)\n",
            "bert.encoder.layer.0.output.dense.bias                        (768,)\n",
            "bert.encoder.layer.0.output.LayerNorm.weight                  (768,)\n",
            "bert.encoder.layer.0.output.LayerNorm.bias                    (768,)\n",
            "\n",
            "==== Output Layer ====\n",
            "\n",
            "bert.pooler.dense.weight                                  (768, 768)\n",
            "bert.pooler.dense.bias                                        (768,)\n",
            "classifier.weight                                          (41, 768)\n",
            "classifier.bias                                                (41,)\n"
          ]
        }
      ],
      "source": [
        "# Get all of the model's parameters as a list of tuples.\n",
        "params = list(model.named_parameters())\n",
        "\n",
        "print('The BERT model has {:} different named parameters.\\n'.format(len(params)))\n",
        "\n",
        "print('==== Embedding Layer ====\\n')\n",
        "\n",
        "for p in params[0:5]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
        "\n",
        "print('\\n==== First Transformer ====\\n')\n",
        "\n",
        "for p in params[5:21]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
        "\n",
        "print('\\n==== Output Layer ====\\n')\n",
        "\n",
        "for p in params[-4:]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "id": "E2i3z5-xWsfs",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "61e1a40e-c7c0-413b-fcec-d861139e58a5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/optimization.py:429: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "# Note: AdamW is a class from the huggingface library (as opposed to pytorch)\n",
        "# I believe the 'W' stands for 'Weight Decay fix\"\n",
        "optimizer = AdamW(model.parameters(),\n",
        "                  lr = 2e-5, # args.learning_rate - default is 5e-5, our notebook had 2e-5\n",
        "                  eps = 1e-8 # args.adam_epsilon  - default is 1e-8.\n",
        "                )\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "id": "Jq8KTv99W4sO"
      },
      "outputs": [],
      "source": [
        "optimizer = torch.optim.AdamW(model.parameters(),\n",
        "                  lr = 2e-5, # args.learning_rate - default is 5e-5, our notebook had 2e-5\n",
        "                  eps = 1e-8 # args.adam_epsilon  - default is 1e-8.\n",
        "                )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "id": "A0z2WMekXI6g"
      },
      "outputs": [],
      "source": [
        "from transformers import get_linear_schedule_with_warmup\n",
        "\n",
        "# Number of training epochs. The BERT authors recommend between 2 and 4.\n",
        "# We chose to run for 4, but we'll see later that this may be over-fitting the\n",
        "# training data.\n",
        "epochs = 4\n",
        "\n",
        "# Total number of training steps is [number of batches] x [number of epochs].\n",
        "# (Note that this is not the same as the number of training samples).\n",
        "total_steps = len(train_dataloader) * epochs\n",
        "\n",
        "# Create the learning rate scheduler.\n",
        "scheduler = get_linear_schedule_with_warmup(optimizer,\n",
        "                                            num_warmup_steps = 0, # Default value in run_glue.py\n",
        "                                            num_training_steps = total_steps)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "id": "wVy7NzjDXq9U"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "\n",
        "# Function to calculate the accuracy of our predictions vs labels\n",
        "def flat_accuracy(preds, labels):\n",
        "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
        "    labels_flat = labels.flatten()\n",
        "    return np.sum(pred_flat == labels_flat) / len(labels_flat)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {
        "id": "1b9UiweLYuae"
      },
      "outputs": [],
      "source": [
        "import time\n",
        "import datetime\n",
        "\n",
        "def format_time(elapsed):\n",
        "    '''\n",
        "    Takes a time in seconds and returns a string hh:mm:ss\n",
        "    '''\n",
        "    # Round to the nearest second.\n",
        "    elapsed_rounded = int(round((elapsed)))\n",
        "\n",
        "    # Format as hh:mm:ss\n",
        "    return str(datetime.timedelta(seconds=elapsed_rounded))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {
        "id": "oXeEO4bwY432",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3baf8021-a349-4d44-cefc-b4fa51cd5455"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "======== Epoch 1 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of    343.    Elapsed: 0:06:19.\n",
            "  Batch    80  of    343.    Elapsed: 0:12:32.\n",
            "  Batch   120  of    343.    Elapsed: 0:18:46.\n",
            "  Batch   160  of    343.    Elapsed: 0:24:58.\n",
            "  Batch   200  of    343.    Elapsed: 0:32:13.\n",
            "  Batch   240  of    343.    Elapsed: 0:38:45.\n",
            "  Batch   280  of    343.    Elapsed: 0:45:10.\n",
            "  Batch   320  of    343.    Elapsed: 0:51:22.\n",
            "\n",
            "  Average training loss: 2.97\n",
            "  Training epcoh took: 0:54:52\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.01\n",
            "  Validation Loss: 2.79\n",
            "  Validation took: 0:01:47\n",
            "\n",
            "======== Epoch 2 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of    343.    Elapsed: 0:06:11.\n",
            "  Batch    80  of    343.    Elapsed: 0:12:23.\n",
            "  Batch   120  of    343.    Elapsed: 0:18:37.\n",
            "  Batch   160  of    343.    Elapsed: 0:24:49.\n",
            "  Batch   200  of    343.    Elapsed: 0:31:01.\n",
            "  Batch   240  of    343.    Elapsed: 0:37:39.\n",
            "  Batch   280  of    343.    Elapsed: 0:43:56.\n",
            "  Batch   320  of    343.    Elapsed: 0:50:09.\n",
            "\n",
            "  Average training loss: 2.72\n",
            "  Training epcoh took: 0:53:40\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.02\n",
            "  Validation Loss: 2.70\n",
            "  Validation took: 0:01:47\n",
            "\n",
            "======== Epoch 3 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of    343.    Elapsed: 0:06:13.\n",
            "  Batch    80  of    343.    Elapsed: 0:12:25.\n",
            "  Batch   120  of    343.    Elapsed: 0:18:37.\n",
            "  Batch   160  of    343.    Elapsed: 0:24:50.\n",
            "  Batch   200  of    343.    Elapsed: 0:31:06.\n",
            "  Batch   240  of    343.    Elapsed: 0:37:20.\n",
            "  Batch   280  of    343.    Elapsed: 0:43:33.\n",
            "  Batch   320  of    343.    Elapsed: 0:49:45.\n",
            "\n",
            "  Average training loss: 2.57\n",
            "  Training epcoh took: 0:53:16\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.03\n",
            "  Validation Loss: 2.61\n",
            "  Validation took: 0:01:48\n",
            "\n",
            "======== Epoch 4 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of    343.    Elapsed: 0:06:14.\n",
            "  Batch    80  of    343.    Elapsed: 0:12:26.\n",
            "  Batch   120  of    343.    Elapsed: 0:18:38.\n",
            "  Batch   160  of    343.    Elapsed: 0:24:50.\n",
            "  Batch   200  of    343.    Elapsed: 0:31:15.\n",
            "  Batch   240  of    343.    Elapsed: 0:37:46.\n",
            "  Batch   280  of    343.    Elapsed: 0:44:18.\n",
            "  Batch   320  of    343.    Elapsed: 0:50:51.\n",
            "\n",
            "  Average training loss: 2.49\n",
            "  Training epcoh took: 0:54:40\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.02\n",
            "  Validation Loss: 2.59\n",
            "  Validation took: 0:01:58\n",
            "\n",
            "Training complete!\n",
            "Total training took 3:43:47 (h:mm:ss)\n"
          ]
        }
      ],
      "source": [
        "import random\n",
        "import numpy as np\n",
        "\n",
        "# This training code is based on the `run_glue.py` script here:\n",
        "# https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128\n",
        "\n",
        "\n",
        "# Set the seed value all over the place to make this reproducible.\n",
        "seed_val = 42\n",
        "\n",
        "random.seed(seed_val)\n",
        "np.random.seed(seed_val)\n",
        "torch.manual_seed(seed_val)\n",
        "torch.cuda.manual_seed_all(seed_val)\n",
        "\n",
        "# We'll store a number of quantities such as training and validation loss,\n",
        "# validation accuracy, and timings.\n",
        "training_stats = []\n",
        "\n",
        "# Measure the total training time for the whole run.\n",
        "total_t0 = time.time()\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "# For each epoch...\n",
        "for epoch_i in range(0, epochs):\n",
        "\n",
        "    # ========================================\n",
        "    #               Training\n",
        "    # ========================================\n",
        "\n",
        "    # Perform one full pass over the training set.\n",
        "\n",
        "    print(\"\")\n",
        "    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
        "    print('Training...')\n",
        "\n",
        "    # Measure how long the training epoch takes.\n",
        "    t0 = time.time()\n",
        "\n",
        "    # Reset the total loss for this epoch.\n",
        "    total_train_loss = 0\n",
        "\n",
        "\n",
        "\n",
        "    # Put the model into training mode. Don't be mislead--the call to\n",
        "    # `train` just changes the *mode*, it doesn't *perform* the training.\n",
        "    # `dropout` and `batchnorm` layers behave differently during training\n",
        "    # vs. test (source: https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch)\n",
        "    model.train()\n",
        "\n",
        "    #train_dataloader = DataLoader(train_dataset, batch_size=32, shuffle=True)\n",
        "\n",
        "    # For each batch of training data...\n",
        "    for step, batch in enumerate(train_dataloader):\n",
        "\n",
        "        # Progress update every 40 batches.\n",
        "        if step % 40 == 0 and not step == 0:\n",
        "            # Calculate elapsed time in minutes.\n",
        "            elapsed = format_time(time.time() - t0)\n",
        "\n",
        "            # Report progress.\n",
        "            print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
        "\n",
        "        # Unpack this training batch from our dataloader.\n",
        "        #\n",
        "        # As we unpack the batch, we'll also copy each tensor to the GPU using the\n",
        "        # `to` method.\n",
        "        #\n",
        "        # `batch` contains three pytorch tensors:\n",
        "        #   [0]: input ids\n",
        "        #   [1]: attention masks\n",
        "        #   [2]: labels\n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "        b_labels = batch[2].to(device)\n",
        "        #b_labels = b_labels.float()\n",
        "        # Always clear any previously calculated gradients before performing a\n",
        "        # backward pass. PyTorch doesn't do this automatically because\n",
        "        # accumulating the gradients is \"convenient while training RNNs\".\n",
        "        # (source: https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch)\n",
        "        model.zero_grad()\n",
        "\n",
        "        # Perform a forward pass (evaluate the model on this training batch).\n",
        "        # The documentation for this `model` function is here:\n",
        "        # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification\n",
        "        # It returns different numbers of parameters depending on what arguments\n",
        "        # arge given and what flags are set. For our useage here, it returns\n",
        "        # the loss (because we provided labels) and the \"logits\"--the model\n",
        "        # outputs prior to activation.\n",
        "\n",
        "        #b_labels = b_labels.float()\n",
        "\n",
        "        loss, logits = model(b_input_ids,\n",
        "                             token_type_ids=None,\n",
        "                             attention_mask=b_input_mask,\n",
        "                             labels=b_labels - 1\n",
        "                             ).to_tuple()\n",
        "\n",
        "        # Accumulate the training loss over all of the batches so that we can\n",
        "        # calculate the average loss at the end. `loss` is a Tensor containing a\n",
        "        # single value; the `.item()` function just returns the Python value\n",
        "        # from the tensor.\n",
        "        total_train_loss += loss.item()\n",
        "\n",
        "        # Perform a backward pass to calculate the gradients.\n",
        "        loss.backward()\n",
        "\n",
        "        # Clip the norm of the gradients to 1.0.\n",
        "        # This is to help prevent the \"exploding gradients\" problem.\n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "        # Update parameters and take a step using the computed gradient.\n",
        "        # The optimizer dictates the \"update rule\"--how the parameters are\n",
        "        # modified based on their gradients, the learning rate, etc.\n",
        "        optimizer.step()\n",
        "\n",
        "        # Update the learning rate.\n",
        "        scheduler.step()\n",
        "\n",
        "    # Calculate the average loss over all of the batches.\n",
        "    avg_train_loss = total_train_loss / len(train_dataloader)\n",
        "\n",
        "    # Measure how long this epoch took.\n",
        "    training_time = format_time(time.time() - t0)\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
        "    print(\"  Training epcoh took: {:}\".format(training_time))\n",
        "\n",
        "    # ========================================\n",
        "    #               Validation\n",
        "    # ========================================\n",
        "    # After the completion of each training epoch, measure our performance on\n",
        "    # our validation set.\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"Running Validation...\")\n",
        "\n",
        "    t0 = time.time()\n",
        "\n",
        "    # Put the model in evaluation mode--the dropout layers behave differently\n",
        "    # during evaluation.\n",
        "    model.eval()\n",
        "\n",
        "    # Tracking variables\n",
        "    total_eval_accuracy = 0\n",
        "    total_eval_loss = 0\n",
        "    nb_eval_steps = 0\n",
        "\n",
        "    # Evaluate data for one epoch\n",
        "    for batch in validation_dataloader:\n",
        "\n",
        "        # Unpack this training batch from our dataloader.\n",
        "        #\n",
        "        # As we unpack the batch, we'll also copy each tensor to the GPU using\n",
        "        # the `to` method.\n",
        "        #\n",
        "        # `batch` contains three pytorch tensors:\n",
        "        #   [0]: input ids\n",
        "        #   [1]: attention masks\n",
        "        #   [2]: labels\n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "        b_labels = batch[2].to(device)\n",
        "\n",
        "\n",
        "        # Tell pytorch not to bother with constructing the compute graph during\n",
        "        # the forward pass, since this is only needed for backprop (training).\n",
        "        with torch.no_grad():\n",
        "\n",
        "            # Forward pass, calculate logit predictions.\n",
        "            # token_type_ids is the same as the \"segment ids\", which\n",
        "            # differentiates sentence 1 and 2 in 2-sentence tasks.\n",
        "            # The documentation for this `model` function is here:\n",
        "            # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification\n",
        "            # Get the \"logits\" output by the model. The \"logits\" are the output\n",
        "            # values prior to applying an activation function like the softmax.\n",
        "            (loss, logits) = model(b_input_ids,\n",
        "                                   token_type_ids=None,\n",
        "                                   attention_mask=b_input_mask,\n",
        "                                   labels=b_labels -1).to_tuple()\n",
        "\n",
        "        # Accumulate the validation loss.\n",
        "        total_eval_loss += loss.item()\n",
        "\n",
        "        # Move logits and labels to CPU\n",
        "        logits = logits.detach().cpu().numpy()\n",
        "        label_ids = b_labels.to('cpu').numpy()\n",
        "\n",
        "        # Calculate the accuracy for this batch of test sentences, and\n",
        "        # accumulate it over all batches.\n",
        "        total_eval_accuracy += flat_accuracy(logits, label_ids)\n",
        "\n",
        "\n",
        "    # Report the final accuracy for this validation run.\n",
        "    avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)\n",
        "    print(\"  Accuracy: {0:.2f}\".format(avg_val_accuracy))\n",
        "\n",
        "    # Calculate the average loss over all of the batches.\n",
        "    avg_val_loss = total_eval_loss / len(validation_dataloader)\n",
        "\n",
        "    # Measure how long the validation run took.\n",
        "    validation_time = format_time(time.time() - t0)\n",
        "\n",
        "    print(\"  Validation Loss: {0:.2f}\".format(avg_val_loss))\n",
        "    print(\"  Validation took: {:}\".format(validation_time))\n",
        "\n",
        "    # Record all statistics from this epoch.\n",
        "    training_stats.append(\n",
        "        {\n",
        "            'epoch': epoch_i + 1,\n",
        "            'Training Loss': avg_train_loss,\n",
        "            'Valid. Loss': avg_val_loss,\n",
        "            'Valid. Accur.': avg_val_accuracy,\n",
        "            'Training Time': training_time,\n",
        "            'Validation Time': validation_time\n",
        "        }\n",
        "    )\n",
        "\n",
        "print(\"\")\n",
        "print(\"Training complete!\")\n",
        "\n",
        "print(\"Total training took {:} (h:mm:ss)\".format(format_time(time.time()-total_t0)))"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#5. Performance On Test Set\n",
        "\n",
        "# 5.1。Data Preparation\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "import pandas as pd\n",
        "\n",
        "# Load the dataset into a pandas dataframe.\n",
        "#df = pd.read_csv(\"/content/preprocessed_data(210).csv\", delimiter='\\t', header=None, names=['sentence_source', 'label', 'label_notes', 'sentence'])\n",
        "df = pd.read_csv(\"/content/preprocessed_data(210).csv\", names=['post_number','title','main_content','comments', 'word_count', 'URL', 'Category', 'Classification'])\n",
        "df = df.dropna(axis=0, how='any')\n",
        "\n",
        "# Report the number of sentences.\n",
        "print('Number of test sentences: {:,}\\n'.format(df.shape[0]))\n",
        "\n",
        "# Create sentence and label lists\n",
        "#sentences = df.sentence.values\n",
        "#labels = df.label.values\n",
        "\n",
        "sentences = df.comments[1:].values\n",
        "labels = df.Category[1:].values\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "labels_array = df.Category[1:].values.astype(float)\n",
        "\n",
        "labels = torch.tensor(labels_array, dtype=torch.long)\n",
        "\n",
        "\n",
        "# Tokenize all of the sentences and map the tokens to thier word IDs.\n",
        "input_ids = []\n",
        "attention_masks = []\n",
        "\n",
        "# For every sentence...\n",
        "for sent in sentences:\n",
        "    # `encode_plus` will:\n",
        "    #   (1) Tokenize the sentence.\n",
        "    #   (2) Prepend the `[CLS]` token to the start.\n",
        "    #   (3) Append the `[SEP]` token to the end.\n",
        "    #   (4) Map tokens to their IDs.\n",
        "    #   (5) Pad or truncate the sentence to `max_length`\n",
        "    #   (6) Create attention masks for [PAD] tokens.\n",
        "    encoded_dict = tokenizer.encode_plus(\n",
        "                        sent,                      # Sentence to encode.\n",
        "                        add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
        "                        max_length = 64,           # Pad & truncate all sentences.\n",
        "                        padding='max_length',  # Use the new padding argument\n",
        "                        truncation=True,  # Explicitly activate truncation\n",
        "                        #pad_to_max_length = True,\n",
        "                        return_attention_mask = True,   # Construct attn. masks.\n",
        "                        return_tensors = 'pt',     # Return pytorch tensors.\n",
        "                   )\n",
        "\n",
        "    # Add the encoded sentence to the list.\n",
        "    input_ids.append(encoded_dict['input_ids'])\n",
        "\n",
        "    # And its attention mask (simply differentiates padding from non-padding).\n",
        "    attention_masks.append(encoded_dict['attention_mask'])\n",
        "\n",
        "# Convert the lists into tensors.\n",
        "input_ids = torch.cat(input_ids, dim=0)\n",
        "attention_masks = torch.cat(attention_masks, dim=0)\n",
        "#labels = torch.tensor(labels)\n",
        "\n",
        "# Set the batch size.\n",
        "batch_size = 16\n",
        "\n",
        "# Create the DataLoader.\n",
        "prediction_data = TensorDataset(input_ids, attention_masks, labels)\n",
        "prediction_sampler = SequentialSampler(prediction_data)\n",
        "prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LRXK3Z6-joHJ",
        "outputId": "b3d36d4f-fb66-4673-9fca-34024662d23f"
      },
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of test sentences: 6,090\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:2619: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "### 5.2 Evaluate the test set\n",
        "# Prediction on test set\n",
        "\n",
        "print('Predicting labels for {:,} test sentences...'.format(len(input_ids)))\n",
        "\n",
        "# Put model in evaluation mode\n",
        "model.eval()\n",
        "\n",
        "# Tracking variables\n",
        "predictions , true_labels = [], []\n",
        "\n",
        "# Predict\n",
        "for batch in prediction_dataloader:\n",
        "  # Add batch to GPU\n",
        "  batch = tuple(t.to(device) for t in batch)\n",
        "\n",
        "  # Unpack the inputs from our dataloader\n",
        "  b_input_ids, b_input_mask, b_labels = batch\n",
        "\n",
        "  # Telling the model not to compute or store gradients, saving memory and\n",
        "  # speeding up prediction\n",
        "  with torch.no_grad():\n",
        "      # Forward pass, calculate logit predictions\n",
        "      outputs = model(b_input_ids, token_type_ids=None,\n",
        "                      attention_mask=b_input_mask)\n",
        "\n",
        "  logits = outputs[0]\n",
        "\n",
        "  # Move logits and labels to CPU\n",
        "  logits = logits.detach().cpu().numpy()\n",
        "  label_ids = b_labels.to('cpu').numpy()\n",
        "\n",
        "  # Store predictions and true labels\n",
        "  predictions.append(logits)\n",
        "  true_labels.append(label_ids)\n",
        "\n",
        "print('    DONE.')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mizq3QF8lJjd",
        "outputId": "25979993-512c-435f-f8de-b549f8b0fb87"
      },
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Predicting labels for 6,089 test sentences...\n",
            "    DONE.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# result is wrong because i set the wrong value\n",
        "\n",
        "df.Category[1:] = df.Category[1:].astype(float)\n",
        "\n",
        "\n",
        "print('Positive samples: %d of %d (%.2f%%)' % (labels.sum(), len(labels), (labels.sum() / len(labels) * 100.0)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1A1M1TA-m6uS",
        "outputId": "dbf37ff5-6f48-4801-b019-dd577a32cdd3"
      },
      "execution_count": 62,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Positive samples: 92551 of 6089 (1519.97%)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import matthews_corrcoef\n",
        "\n",
        "matthews_set = []\n",
        "\n",
        "# Evaluate each test batch using Matthew's correlation coefficient\n",
        "print('Calculating Matthews Corr. Coef. for each batch...')\n",
        "\n",
        "# For each input batch...\n",
        "for i in range(len(true_labels)):\n",
        "\n",
        "  # The predictions for this batch are a 2-column ndarray (one column for \"0\"\n",
        "  # and one column for \"1\"). Pick the label with the highest value and turn this\n",
        "  # in to a list of 0s and 1s.\n",
        "  pred_labels_i = np.argmax(predictions[i], axis=1).flatten()\n",
        "\n",
        "  # Calculate and store the coef for this batch.\n",
        "  matthews = matthews_corrcoef(true_labels[i], pred_labels_i)\n",
        "  matthews_set.append(matthews)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kfac5DJOtlD7",
        "outputId": "8f0dcae7-df33-4343-e95b-412cb98d6025"
      },
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Calculating Matthews Corr. Coef. for each batch...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a barplot showing the MCC score for each batch of test samples.\n",
        "ax = sns.barplot(x=list(range(len(matthews_set))), y=matthews_set, ci=None)\n",
        "\n",
        "plt.title('MCC Score per Batch')\n",
        "plt.ylabel('MCC Score (-1 to +1)')\n",
        "plt.xlabel('Batch #')\n",
        "\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 681
        },
        "id": "tGgph71Ytquj",
        "outputId": "b43f2c3d-7b49-42e8-f4ed-f51e7ebc5500"
      },
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-64-a1af80ea924d>:2: FutureWarning: \n",
            "\n",
            "The `ci` parameter is deprecated. Use `errorbar=None` for the same effect.\n",
            "\n",
            "  ax = sns.barplot(x=list(range(len(matthews_set))), y=matthews_set, ci=None)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1200x600 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAABDsAAAI/CAYAAAB58tIOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAACi2UlEQVR4nOzdd3hUZf7//9dMCiEESEIPfRFFEHSVuiqrwCqKShFh7SworthFXV3Fz+eroqJrQVn4KEgRy4ICFtSAiG1VqgIiICIEkKqQkJBCkpnz+yO/GVNmkjnT58zzcV1cF5lzzn3u0+/zPnexGYZhCAAAAAAAwCLskc4AAAAAAABAMBHsAAAAAAAAlkKwAwAAAAAAWArBDgAAAAAAYCkEOwAAAAAAgKUQ7AAAAAAAAJZCsAMAAAAAAFgKwQ4AAAAAAGApBDsAAAAAAIClEOwAAABAXLr//vt1yimn6P777490VgAAQZYY6QwAABDNXnzxRU2bNs3997PPPqshQ4bUusz48eP1+eefu//+5JNP1KZNG6/zHzlyRIsWLdI333yjn3/+WXl5ebLb7crMzFSXLl305z//WRdddJEaNWrkNQ2Hw6Fly5bps88+08aNG3XkyBGVlJSoYcOG6tChg3r27KlLL71UJ598somt/92OHTv0n//8R2vWrNG+fft04sQJpaenq2nTpurUqZN69uypvn37qmPHjn6lj+i0ePFiPfDAAx6n1atXT02bNlX37t01YsQI/fnPfw76+vPz8zVv3jxJ0vXXX1/rNQAAQGUEOwAAMGHx4sW1BjsOHTqk//73vz6lZRiGXnrpJf3f//2fiouL3b+npqbKZrNp37592rdvnz755BM9/fTTuv/++zVy5Mga6WzYsEH/+Mc/lJOT4/4tKSlJDRo0UF5enr799lt9++23evnll3XBBRfomWeeUXJyss/bPGvWLD333HMqLy93/9aoUSMVFBTo119/1datW7V06VL17t1b8+fP9zldxJaMjAwlJCRIqjh3jx075j5Hs7OzNWLECD3xxBNBXWd+fr472Dh8+HCCHQAAnxHsAADABxkZGTpx4oS+/vprHTx4UC1btvQ437vvviuHw6HWrVtr3759XtMzDEP33nuv3n//fUnS6aefrhtuuEF9+/Z1v9AdP35cq1at0ttvv61PP/1UK1eurBHsWLlype644w6VlpYqPT1d48aN0wUXXKAOHTpIqqjxsWXLFi1fvlxvvPGGli9frpKSEp+DHcuXL9fTTz8tSerVq5duvvlm9ezZU/Xq1ZNUEdxZt26dli1bpuPHj/uUJmLT22+/XaWGktPp1Pbt2/XUU0/pq6++0uLFi3X22WfrkksuiWAuAQCoQJ8dAAD4IDU1VRdeeKGcTqcWL17sdb5FixZJkkaMGFFrejNnznQHOq6//notWLBAF1xwQZUv12lpaRo0aJD+7//+T6+99lqNAEtOTo7uvfdelZaW6qSTTtK7776r8ePHuwMdkpSQkKDu3btr4sSJ+uSTTzRw4EBT2z179mxJ0sknn6y5c+fq7LPPdgc6JKlFixYaMmSIXnjhBU2fPt1U2ohtdrtdXbp00bRp09SwYUNJ0ooVKyKcKwAAKlCzAwAAH40YMUJLlizRkiVLNGHChBrT161bp5ycHLVt21Y9e/b0ms7Ro0c1Y8YMSVK/fv30wAMPyGaz1bruXr166ayzzqry2/PPP6/jx4+rXr16mjZtmtfaJi7p6emaPn26DMOodb7Ktm3bJknq37+/EhNrLzakpKR4nVZUVKQFCxbok08+0U8//aTCwkJlZmaqXbt2GjBggC677DI1bdq0xnKrV6/W66+/ru+++065ublq0KCBunTpossuu0zDhg1zN6uozNXPiqtZzbJly7RgwQJt3bpVubm5uuWWW3Tbbbe55z969KjmzZunzz//XHv37lVpaamaN2+uPn366G9/+5s6d+7s6+6qku/rrrtOkvTjjz/q+++/18yZM/Xtt9/q2LFjatmypQYNGqSbb7651qYZpaWleuutt5Sdna3t27ersLBQjRs3Vo8ePfTXv/7Vaz8Zp5xyiiTp1Vdf1UknnaSXX35Zn332mQ4ePKiSkhL9+OOPprfJm9TUVLVr104//PCDioqKakx3Op1avXq1PvnkE23atEkHDx7U0aNH1aBBA3Xu3FlDhgzRyJEjlZSUVGW5a6+9VmvWrHH/XT1Q56nZVGlpqd59911lZ2dr69atys/PV3p6ulq3bq1zzz1XQ4cOVdu2bb1uS3Z2tl5//XX9+OOPOnHihDp06KARI0bo2muvld3ON0IAiCUEOwAA8FGvXr3Url077dmzR2vXrlWvXr2qTHfV+Bg+fHitwYvFixe7XwpvvfXWOgMdLpVftn777TctW7ZMknTppZea6hjU1/VVdvDgQdPLuPzwww+65ZZbdODAAUkV29GoUSPl5ubq0KFDWrt2rex2u8aMGVNluSeeeEJz585157lhw4YqKCjQqlWrtGrVKr333nv697//rbS0NK/rfvLJJzVnzhzZbDY1atSoxgvr119/rTvuuEP5+fmSKvo6SUpK0i+//KJffvlF7733nh577DENGzbM7+1fsWKF7rzzTpWVlSktLU2GYWjPnj2aPXu2li1bpldffdVjB7b79u3TTTfdpJ9++sm9D9LS0vTbb79p5cqVWrlypf7617/q//2//+d13Xv27NHdd9+t3377TfXq1aszYOWP4uJi7dmzR5I8nof79++vcmxTU1OVkpKivLw8rV27VmvXrtXSpUv1yiuvVAmYNW7cWBkZGcrNzZVUtc8Q1/TK9u7dqwkTJmj79u2S5D7mx48f14YNG7RhwwYdO3ZMDz74oMfteOSRR/T666/LbrcrLS1NJSUl2rZtmx5//HFt2bJFU6ZM8W8HAQAigmAHAAA+stlsGj58uKZOnapFixZVCXYUFRXpo48+kt1u14gRI9wvf5588803kqTMzMxaa4DUZvXq1XI6nZKkv/zlL36l4Yvu3btrzZo1ys7O1nnnnachQ4aY+sJ94MABjRs3Trm5uWrVqpXuu+8+nX/++apfv74Mw9DPP/+s7OxsZWZmVlnutddecwc6Ro8erdtuu03NmjVTUVGRFi5cqKefflqrVq3SpEmT9Nxzz3lc9+bNm7VmzRrdeOONGjt2rDIzM1VaWqpff/1VUkWNi5tvvlklJSUaNWqUxowZow4dOighIUH79+/XzJkz9cYbb+jBBx9Up06d1L17d7/24f33368//vGP+t///V916tRJ5eXlWr58uf73f/9X+/bt05133qkFCxZUeZEvKirSDTfcoJ07d6p379667bbbdMYZZyg5OVkFBQVatGiRpk6dqv/85z/6wx/+oOuvv97juh9//HG1aNFC//rXv9SnTx/Z7Xbt2rXLr+2ozjAMbd++XU8//bQKCgqUkpKiq6++usZ8iYmJuvTSS3XxxRfrzDPPVHp6uiSpsLBQy5Yt03PPPad169bpueeeqzLyy7Rp0/TLL7+4a3RU7zOksuPHj+uGG25QTk6OGjdurHvuuUcXXXSRu3nN3r17tWLFCq+BvpUrV6qoqEgPPPCARo4cqbS0NOXm5uqZZ57RW2+9pXfeeUfDhg1Tv379AtllAIAwoj4eAAAmDB8+XHa7XcuWLVNhYaH7948++khFRUXq16+fWrVqVWsaO3bskCSdeuqpfufD9bU/0HTqcttttykxMVHl5eW655571L9/f915552aNWuWVq1a5bHZQmXPPvuscnNzlZ6erjfffFMXX3yx6tevL6kieHTSSSfp1ltv1WWXXeZepqSkRC+++KIk6ZJLLtEjjzyiZs2aSaqoFTBmzBjdf//9kqQPP/xQmzdv9rjuoqIi/e1vf9M999zjDqYkJyerdevWkioCASUlJbrpppv06KOPqlOnTu6AQ1ZWlv7nf/5H1157rcrLy93NjvzRpEkTzZw5U506dZJU8fJ/8cUX6/nnn5ckff/991q+fHmVZebMmeMOdMyePVu9e/d2dyrbsGFDjRkzRk899ZQkacaMGVVGyqnMbrdr7ty56tevnztI5e/wwCNHjtTZZ5/t/te9e3dddtllWrNmjQYNGqQFCxaoXbt2NZZr2bKl/vWvf2nAgAHuQIckNWjQQCNGjHD39bJw4UKdOHHCr7zNmjVLOTk5Sk5O1ty5czVq1Ch3oEOS2rZtq7/97W81ag+5HDt2TI888ojGjBnjrimUkZGhxx57TN26dZMkffDBB37lDQAQGQQ7AAAwoVWrVvrTn/7krsnh4mrCcvnll9eZRl5enqSa1fDNcKUhqcoLZLD17t1bs2bNcr8g//rrr/roo4/09NNP6/rrr1fv3r01fvx4rV27tsaylffR+PHj6wwCuXz11Vfu7bv11ls9znPVVVe5AyBLly71OI/dbteNN97ocdovv/yiVatWKTExUWPHjvWaF1fzlW+++UYOh8On/Fd3ww03eOzP5E9/+pP++Mc/SqoI2lTm6uh2zJgxNfqycBk0aJC7BsIPP/zgcZ6hQ4fW2ZeLr3Jzc/Xbb7+5/5WVlUmq6CcjPz9fhw8f9ivd7t27q0mTJioqKtLWrVv9SsO1v6644gp17drV9PKtWrXS8OHDPU4bMGCAJAW1nxMAQOjRjAUAAJNGjBih//73v1q0aJFGjhyp3bt3a926dWrcuLEGDRoU6ewFXb9+/fThhx9q3bp1+u9//6uNGzdq27ZtysvLU1lZmT7//HN9/vnnmjBhgu644w73cps3b3a/EJ9//vk+r89VU6NVq1ZeayEkJCSob9++ev/9973W7GjXrp2aNGnicdq3334rqaLzzCFDhnjNiyvAUVRUpLy8PK/p1aZv3761Tvvuu++qbMOhQ4fcwxY/+OCDevjhh70u76pZs2/fPp1++uk1pp955pmm8+vNJ598UqUZSWlpqfbu3avFixdr9uzZWrdunR566CGPTVlKS0u1aNEiffzxx9q+fbv73KnOn75h9u3b5w60mDnPKuvevbvXJi4tWrSQVFH7AwAQOwh2AABg0l/+8hc1btxY3377rXJycrRkyRJJ0pAhQ6oMy+pNenq6Dh48GNDLU+XaHHl5ee4XslCx2+3q3bu3evfu7f7t559/1gcffKA5c+aoqKhI06dPV48ePdwvnL/99pt7XlfTEV8cOXJEkurcJleNBdf81dUWmHC9HDudzir5rE1xcbFP81VX23a4plXehkOHDrn/7+qcsy4lJSUef/cnOOOr5ORkderUSffee68cDofmzJmjxx9/XOeee26V5ixHjhzRmDFj3B2HSlK9evWqdDh69OhROZ1Ov/Zx5eOXlZXl17Y0aNDA6zRXHr01FQIARCeCHQAAmJScnKwhQ4bojTfe0FtvveVuRjFixAiflj/ppJN08OBBv6vsS6oyHOrWrVtDHuzwpFOnTrr99tvVq1cv/e1vf5NhGHrrrbf8/roebJ6GpXVxde7atGlTffXVV+HKkk9ceZMqmre4+vrwR7iGS73iiis0Z84clZeXKzs7W+PHj3dPe/zxx7V9+3alp6frvvvuU//+/d1NkFz+/Oc/6+DBg6aGRXbxZ3QhAID10WcHAAB+cAU25s2bp4MHD+rkk0/2ebQO14gOR48e1bp16/xav2tkDUn6+OOP/UojWPr166f27dtLUpWRPiq/0LqaZfjCVRuhriYNrun+1F5o2rSppIqaE3V1shqoyjU1vE2rvA2uvEkVw7bGgso1Kn755Rf3/8vKytzn58MPP6zLL7+8RqDD4XD4XIPFk1jcXwCA0CPYAQCAH7p3766TTz7Z3e+ALx2TuowYMcI9Ism0adN8/ppd+Yt/06ZNdcEFF0iq6KDTzHCi/nw9r0tqaqokuUcMkaTTTjvN3bnmp59+6nNap512mqSKYIa37XI4HFq9erUk+TUkrKsvC4fDoS+++ML08masWrXK6zTXNri2WZLatGnjrqljZr9FUuXAlOvclioCeq4RVryNGrR+/Xqvo7BUrpni7bzNysqKuf0FAAg9gh0AAPjpnnvu0dixYzV27NgqQ6fWJTMzUzfffLOkilE+nnzyyToDEOvXr9fkyZOr/HbnnXcqNTVVJSUluu2222qtQSBVdLB42223qaCgwOe8/ve//60zb9u2bdO2bdskqcpIGPXr13d3/vnyyy/rwIEDPq3z7LPPdvdJMm3aNI/z/Oc//3H3u1FbB6PedOjQwd3/yHPPPVfnPqk8+o1Zs2fP9vgyv2rVKndHqRdddFGVaaNGjZIkvf3229qyZUvI8hYslUfEqRy4SUtLczczcZ0jlZWXl+u5557zmq5rGFhJtR6jkSNHSpLeeuutOvcXACA+EOwAAMBPf/7zn/WPf/xD//jHP5SZmWlq2fHjx+viiy+WJM2dO1dXXnmlPv74Yx0/ftw9z/Hjx/Xpp5/q1ltv1dVXX10jWNCxY0c9/fTTSkpK0k8//aShQ4fq5Zdf1u7du93zOBwObdmyRVOnTtWgQYO0fPlyU/mcOHGiBg8erH//+9/atGmTSktL3dN+/fVXzZ07V2PGjJHT6VRiYqKuv/76KsvfddddysjIUF5enq688kp9+OGH7s40DcPQ9u3bNWXKFL3zzjvuZVJSUnTbbbdJqniJfvjhh92dUBYXF+vVV1/VE088IUm6+OKLq7xcmzFp0iSlpqYqJydHo0aN0ooVK6oEJQ4dOqR33nlH119/vf71r3/5tQ6pYj+NHz9eO3fulCR3vxaukWu6devmrqXj8re//U0nn3yyTpw4oeuuu06vvfZalaYe+fn5+vzzz3Xfffd5HP0kXI4dO6a5c+fqpZdeklTREW3lbWnQoIG7Fs2TTz6pb775xl1Dafv27Ro/frw2b97srhlUXaNGjdy1NhYvXuy1k9CxY8eqQ4cOKi0t1ZgxY7Rw4cIq19KePXs0bdo0vfLKK4FvNAAgJtBBKQAAEWCz2fTss8+qU6dOmjlzpr777jvdeuutkn4fGaKwsNA9f3p6eo0XYkkaNGiQ5s2bpwceeEC7d+/WM888o2eeeUZJSUlq0KCB8vPz3S+XNptNl1xySZVmBnVJSkpSTk6OXnjhBb3wwguy2+1q2LChiouLqwQ+GjRooMcff1xdunSpsnzLli31yiuv6Oabb9aBAwd01113KSEhwZ2GK7jwwAMPVFnummuu0d69ezV37lwtWLBACxcuVKNGjVRYWOh+4e3Tp48effRRn7elupNPPlmzZs3SHXfcoZ07d+qWW25x562kpKTKCCdt27b1ez1PPvmk7rzzTl100UVq2LChTpw44d53WVlZmjp1qhITqxbJGjRooFmzZun222/Xhg0b9Oijj+qxxx5Tw4YN5XQ6q7zIu/pLCbWRI0dW6fS1vLxcx44dc9f8ycrK0ksvvVRjRKJ//vOfuvbaa3Xo0CGNGTNGycnJSkpKUmFhoRITEzV58mS98MILXvtO+etf/6qpU6dq/vz5WrBggZo0aSK73a7TTz/dXSskLS1Ns2bN0t///nft2LFDkyZN0v/8z/+oUaNGOnHihHuUl+uuuy4UuwYAEIUIdgAAECE2m0233nqrRo8ercWLF+vrr7/Wzp07lZeXJ7vdrtatW+vUU0/Veeedp4suuqhKlf7KzjrrLH300UfKzs7Wp59+qk2bNunIkSMqLCxU48aN9Yc//EG9evXS0KFD9Yc//MFUHrOzs/Xf//5Xq1ev1pYtW7Rnzx7l5+fLZrOpadOm+sMf/qCzzz5bI0eOrNJRZGXdunXThx9+qDfeeEOffPKJdu7cqcLCQjVt2lRt27bVwIEDdemll9ZY7oEHHtD555+vN954Q99++63y8vLUoEEDdenSRUOHDtWwYcNqHXHFF2eddZays7O1cOFCrVy5Uj/99JMKCgpUr149derUSd26dVP//v01cOBAv9cxaNAgvfnmm5o5c6bWr1+vkpIStWnTRhdccIH+/ve/q3Hjxh6Xa9Gihd544w1lZ2dr6dKl2rx5s3Jzc93nxsknn6x+/frVaAITKtU7EU1MTFR6ero6d+6sgQMH6oorrvA4hOtpp52mt956S9OmTdOqVat0/PhxNWjQQP3799fYsWPVo0cPvfDCC17X+/e//11paWl69913tXPnTveoLdWHM27btq2WLFmit99+Wx999JG2b9+uwsJCZWRkqEuXLurfv7+GDh0anJ0BAIh6NiMUvZQBAADEsdWrV7trEfz4448Rzg0AAPGHPjsAAAAAAIClEOwAAAAAAACWQrADAAAAAABYCsEOAAAAAABgKXRQCgAAAAAALIWaHQAAAAAAwFIIdgAAAAAAAEtJjHQGED6GYcjppNUSAAAAACD07HabbDZbRNZNsCOOOJ2Gjh4tjHQ2AAAAAABxIDOzgRISIhPsoBkLAAAAAACwFIIdAAAAAADAUgh2AAAAAAAASyHYAQAAAAAALIVgBwAAAAAAsBSCHQAAAAAAwFIIdgAAAAAAAEsh2AEAAAAAACyFYAcAAAAAALAUgh0AAAAAAMBSCHYAAAAAAABLIdgBAAAAAAAshWAHAAAAAACwFIIdAAAAAADAUgh2AAAAAAAASyHYAQAAAAAALIVgBwAAAAAAsBSCHQAAAAAAwFISI52BSFq1apXmzJmjjRs3qqioSFlZWRo8eLDGjx+v1NRUU2n98ssv+uabb/T9999r8+bN2r59u8rKyjR8+HA9+eSTdS5fWFiol19+WcuWLdP+/fuVmpqq008/XWPHjlWfPn383UQAAAAAiBi73Sa73San05DTaUQ6O4gjcRvsmD9/viZPnizDMNSyZUu1atVKO3bs0IwZM7R8+XK98cYbSk9P9zm9efPm6dVXX/UrL0ePHtVVV12lXbt2KTk5WSeddJKOHj2qzz77TJ9//rkmTZqkq6++2q+0AQAAACAS7Hab0tNTlZBgl8PhVF5eEQEPhE1cNmPZvHmzHn/8cUnSI488os8++0xLlizRihUr1K1bN/3888+aNGmSqTQzMjJ03nnn6bbbbtPLL7+skSNH+rzsgw8+qF27dqlbt25asWKFlixZos8++0yPPPKIDMPQ5MmTtXXrVlP5AQAAAIBIstttSkiwa8mK75WQYJfdbot0lhBH4jLYMX36dDmdTg0dOlSjR4+WzVZx0bVo0ULPPvus7Ha7li9frm3btvmc5oQJE/TSSy/p1ltv1Z///Gc1atTIp+W2bNmilStXym6367nnnlOLFi0kSTabTaNHj9bQoUPlcDg0ffp08xsKAAAAABH2a15hpLOAOBR3wY7CwkJ9+eWXkqRRo0bVmN6hQwf17dtXkpSdnR3y/CxbtkyS1LdvX7Vv377G9NGjR0uSPv/8cxUVFYU8PwAAAAAAxLq4C3Zs3bpVpaWlSk5OVo8ePTzOc9ZZZ0mSNm7cGPL8bNiwQZLUs2dPj9N79Oih5ORknThxgqYsAAAAAAD4IO6CHbt27ZIkZWVlKSkpyeM87dq1qzJvKOXk5FRZZ3VJSUlq1apV2PIDAAAAAECsi7vRWI4dOyZJaty4sdd5XNNc80ZLfvLz8wNeX2Ji3MW3AAAAAERAQoK91r+BUIq7YMeJEyckyWutDklKTk6uMm+05KekpCSgddntNmVkNAgoDQAAAADwR6NG9SOdBcSRuAt21KtXT5JUVlbmdZ7S0tIq84Y6P8XFxT7lJyUlJaB1OZ2G8vPp5BQAAABA6CUk2KsEOPLzi+VwOCOYI4Rbo0b1I1ajJ+6CHb40UfGlaUmwNGrUSMXFxT7lx9fhbGtTXs7NBQAAAED4ORxO3kcQNnHXaKpDhw6SpP3793utTbFnz54q84YjP7t37/Y4vaysTPv37w9bfgAAAAAAiHVxF+w49dRTlZSUpNLSUm3atMnjPOvXr5cknXHGGSHPj2sdrnVWt2nTJpWVlalevXo69dRTQ54fAAAAAABiXdwFO9LS0nTOOedIkhYuXFhjek5OjlatWiVJGjx4cMjzc+GFF0qSVq9e7bF2x4IFCyRJ/fv3V4MGdC4KAAAAAEBd4i7YIUkTJkyQzWbTu+++qwULFsgwDEnS4cOHdffdd8vpdGrQoEHq0qVLleUGDBigAQMGKDs7O2h56datm84//3w5HA7dddddOnz4sCTJMAwtWLBA7777rux2u26++eagrRMAAAAAACuzGa43/Tgzd+5cPfnkkzIMQ61atVJGRoZ27Nih0tJSdezYUW+88YYyMzOrLHPKKadIkp544gmNGDGiyrT169drwoQJ7r9LSkpUUlKi5ORkpaamun9/+OGHNWTIkCrLHj16VFdeeaVycnKUnJysk046Sbm5uTpw4IBsNpsefPBBXXvttQFvs8Ph1NGjhQGnAwAAAAB1SUy0KyOjgV5+e5XGj+yr3NxCOiiNM5mZDRiNJdzGjBmjU045RbNnz9amTZt05MgRZWVlafDgwRo/frzpJiPl5eXKy8ur8Xtpaal76FhJOnHiRI15MjMztWjRIs2cOVPZ2dnasWOHUlNT1b9/f40bN059+/Y1vX0AAAAAAMSruK3ZEY+o2QEAAAAgXKjZgUjW7IjLPjsAAAAAAIB1EewAAAAAAACWQrADAAAAAABYCsEOAAAAAABgKQQ7AAAAAACApRDsAAAAAAAAlkKwAwAAAAAAWArBDgAAAAAAYCkEOwAAAAAAgKUQ7AAAAAAAAJZCsAMAAAAAAFgKwQ4AAAAAAGApBDsAAAAAAIClEOwAAAAAAACWQrADAAAAAABYCsEOAAAAAABgKQQ7AAAAAACApRDsAACEnN1uU2KiXXa7LdJZAQAAQBxIjHQGAADWZrfblJ6eqoQEuxwOp/LyiuR0GpHOFgAAACyMmh0AgJCy221KSLBryYrvlZBA7Q4AAACEHsEOAEBY/JpXGOksAAAAIE4Q7AAAAAAAAJZCsAMAAAAAwoiOu4HQo4NSAAAAAAgTOu4GwoOaHQAAAAAQJnTcDYQHwQ4AAAAACDM67gZCi2AHAAAAAACwFIIdAAAAAADAUgh2AAAAAAAASyHYAQAAAAAALIVgBwAAAAAAsBSCHQCAgNjtNobNAwAAQFQh2AEA8JvdblN6eqrS01MJeAAAACBqJEY6A0AscX3BdjoNOZ1GpLMDRJzdblNCgt39f64LAAAARAOCHYCPXF+wExLscjicyssr4sUOAAAAQccHNiBwNGMBfOT6gr1kxfdKSLBTZR8AAABB5/rAlpHRoNZmona7TYmJlEkBbwh2ACb9mlcY6SwAQNSjEA4A/vHlA5uvAREgnhHsAAAAQRXuQjiBFQBWVNsHNmocA3Uj2AEAQC14kTYvnIVwvm4CiGfUOAa8I9gBAIAXvEgHJhyFcL5uAgAATwh2AADgBS/SsYOvmwAAoDKCHQAQQjSBsAZepAFEC54rAOCbxEhnAACsytUEIiHBLofDqby8IjmdRqSzBQCIUTxXAMB31OwAgBChCQQAIJh4rgCA7wh2AECI0QQCABBMPFcAoG4EOwAAAAAAgKUQ7AAAAAAAAJZCsAMAAAAAAFgKwQ4AAAAAAGApBDviDOOyAwAAAACsLjHSGYikVatWac6cOdq4caOKioqUlZWlwYMHa/z48UpNTfUrzWXLlum1117Ttm3bVFZWpvbt2+uyyy7Tddddp6SkpBrz//LLLxo4cGCtaZ5++ulauHChX/mpzG63KSOjAeOyAwAAAAAsLW6DHfPnz9fkyZNlGIZatmypVq1aaceOHZoxY4aWL1+uN954Q+np6abSnDJlimbPni1JateunerXr6+ffvpJTz31lD799FPNnj1bycnJXpc/88wzPf7euXNnU/nwxmazacmK7zV8UHfZ7TaCHQAAAAAAS4rLYMfmzZv1+OOPS5IeeeQRjRo1SjabTYcOHdLNN9+sH374QZMmTdKLL77oc5off/yxO5jx/PPPu2tr/Pzzzxo/frzWrl2rZ599Vvfff7/XNN58883ANswHjMsOK7Lbbe4AHkE8AAAAAHHZZ8f06dPldDo1dOhQjR49WjZbRR8WLVq00LPPPiu73a7ly5dr27ZtPqc5bdo0SdKNN95YpVlKp06d9Nhjj0mSXn/9dR09ejSIWwLAbrcpPT1VGRkNlJ6eSp80AAAAAOIv2FFYWKgvv/xSkjRq1Kga0zt06KC+fftKkrKzs31KMycnxx0YGT16dI3p/fr1U/v27VVaWqpPPvnE36wD8MButykhwa4lK75XQgId8AIAAACIw2YsW7duVWlpqZKTk9WjRw+P85x11ln6+uuvtXHjRp/S3LBhgySpbdu2atGihdc0d+/erY0bN+qKK67wOM9jjz2mnTt3ymazqXXr1jrnnHM0aNAg2e1xF5OSRNMEmBPPTbS4VgAAAICq4i7YsWvXLklSVlaWx9FRpIrORSvPW5ecnJwqy/mb5vz586v8vWDBAp166ql68cUX1bZtW5/yYhWupgkJCXZGj4lzvMjXjmsFAAAAqCnugh3Hjh2TJDVu3NjrPK5prnmDmWZ+fn6V3xMTE3XZZZdpyJAhOumkk9S8eXPl5ubq888/1/PPP6+tW7dq3LhxWrx4sdLS0nzKjy8SEqK7tkhCgt3dNGH4oO5KSkqQw+GMeJ5q+xvBZ7PZ1LBhivtFvqCgRIZR9UU+mo9LOPIW6Wul8jZ5275oPkZ1ieW8R1I49xvHCPGE890a6jqOvhznWDkXYiWfsKa4C3acOHFCkrzW6pDkHh7WNW8w0ywpKanye8uWLfX0009X+a1FixYaNWqU+vTpoxEjRmj37t169dVXNWHCBJ/y44tGjeoHLa1QcjVNiMb8RmOerMr1Ip+enlrnvNF8XEKZt2i4VnxddzQfo7rEct4jKZz7jWOEeML5bg11HUdfjnOsnAuxkk9YQ9wFO+rVqydJKisr8zpPaWlplXmDmWZKSopPaUpS+/btdeWVV2rmzJn6+OOPgxrsyM8vjnhNidokJNir3AyjIb/RmCerc+1z14u8p30ezcclHHmL9PZXXr+3dUc6j4GI5bxHUjj3G8cI8YTz3RrqOo6+HOdYORe85dNm+72ZcvVau7CWRo3qR6xGT9wFO3xpouJLs5TKGjVq5HOarnl99cc//lHS7/2CBIvD4VR5efTdEL2JxvxGY56szpd9Hs3HJRx5i+T2+7ruaD5GdYnlvEdSOPcbxwjxhPPdGlx9bXnrnyzWyz+VORxOOZ2G0tPr098YQi7uGk116NBBkrR//36vNTH27NlTZd66dOzYUZK0e/dur/OYTdPF1TTG4XCYWg4AAABA9GvYMEUZGQ2Unp4qu90W6eyEnN1uc/c3lpBgj4ttRmTEXbDj1FNPVVJSkkpLS7Vp0yaP86xfv16SdMYZZ/iU5umnny5J+uWXX3To0KGgpOny008/Saro2wMAAACAtcTri7+rmTIQKnEX7EhLS9M555wjSVq4cGGN6Tk5OVq1apUkafDgwT6l2bFjR5188smSKoaLre6bb77R7t27lZSUpIEDB/qc18LCQr3xxhuSpLPPPtvn5QAAAADEDl78geALSp8du3bt0oYNG3T48GEdPXpUJ06cUHp6ujIzM9WpUyedeeaZql8/enrenTBhgj777DO9++67OvPMMzVq1CjZbDYdPnxYd999t5xOpwYNGqQuXbpUWW7AgAGSpPvuu69GIOTWW2/V7bffrpkzZ+q0005zz7tz50499NBDkqSrrrpKmZmZVZabNGmSzj33XJ133nnuEVsk6eeff9ZDDz2kX375RampqRo3blzQ9wMAAAAAAFbkd7Dju+++08KFC/Xll1/qyJEjtc6bkJCgbt266dJLL9XQoUPVsGFDf1cbFD169ND999+vJ598Ug8//LBmzJihjIwM7dixQ6WlperYsaMeffTRGsvt27dPklRUVFRj2oUXXqjrr79e8+bN080336x27dopNTVVP/30kxwOh8466yxNnDixxnKbNm3SwoULlZSUpHbt2iktLU25ubnuPj4aN26s559/Xm3atAnyXgAAAAAAwJpMBzveffddzZo1Szt27KgyTFBqaqrS09OVnp6uevXq6dixYzp27Jhyc3NVXl6ujRs3atOmTXrmmWc0ZMgQ3XLLLWrVqlVQN8aMMWPG6JRTTtHs2bO1adMmHTlyRFlZWRo8eLDGjx+vBg0amE7zn//8p/74xz/qjTfe0NatW3X48GF16tRJl112mcaMGePubLSym266SV9++aU2b96s3377Tbt371ZKSoq6deum/v376+qrr1azZs2CsckAAABRy27/fShKRmYAAATK52DH6tWrNWXKFG3dulWGYahx48a68MIL1bNnT51++ulq3769x+UKCwu1efNmbdy4UStXrtSGDRv09ttv6/3339d1112nm266SWlpaUHbIDP69eunfv36+Tz/jz/+WOc8F110kS666CKf07z44ot18cUX+zw/AACA1djtNqWnpzIUJQAgaHwOdlx//fWSpHPOOUd//etf9ec//9ljTYXqGjRooD59+qhPnz4aP3689u7dq3fffVevvfaaZs2apZSUFN1yyy3+bwEAAABiWuWhKIcP6u6u4QEAgL98Dnacc845uu2229zDrPqrbdu2uvXWWzVu3Di99tprSk1NDSg9AAAAWAMjUgAAgsXnYMesWbOCuuL69evrxhtvDGqaAOAr1zj2fDkEAAAArMce6QwAQLi52oanp6e6gx4AAACAJ3a7TYmJdsqNMcbvoWcBIFa52oa7/k/tDgAAAHhCB8qxi5odAAAAAAB4ULkD5YQEanfEkrAFO44fP67rrrvOPaoLEA2okgYAAACgLnSgHHvC1oylrKxMa9askc3GSyWiA1XSAAAAAMCaaMaCuEWVNAAAAACwJoIdiHtUSQMAAAAAazHVjOWBBx7we0WlpaV+LwsAAAAAAOArU8GOJUuW0OcGAAAAAACIaqaCHa5AR5cuXZSWlmZqReXl5fruu+9MLQMAAAAAAGCWqWBHu3bttGfPHo0ZM0ZDhw41taLc3Fz169fP1DIAAAAAAABmmeqgtFu3bpKkLVu2hCQzAAAAAABIFaMnMmIi/GU62GEYhn744YdQ5QcAAAAWY7fblJgYu8O8x3r+ET+sdK7a7Talp6cqPT3VEtuD8DPVjKVr166SpG3btpleUVJSknr16mV6OSDauSLOTqchp9OIdHZiDvvPHPYXgFjjemFJSLDL4XAqL68opu5fsZ5/xA+rnat2u00JCXb3/2N5WxAZpoIdvXv31ieffOLXitLS0jR//ny/lgWildUeKuHG/jPH0/4CgGjnemFZsuJ7DR/UPeZeWmI9/4gfnKtAVaaasSQkJKh169Zq3bp1qPIDxJTKD5WEBGtUGQwn9p857K/QsFKVXyCa/ZpXGOksBCTW84/4wbkKVDBVswOAZzxUAsP+M4f9FTzULgIAALAmUzU7AACwEmrLAAAAWBPBDgA+oao/rIzaMgAAANYS9GDHnDlzNG3atGAnCyCCXFX9MzIaMPwXAAAAgKgX9GDHK6+8on//+9/BThZABFHVHwCA6BTOmpfU8gQQS+igFIhBdrvNPZxYODtTrF7VP1L5AAAAks1mU3p6/bB0skyHzgBiDX12ADEmWpqUREs+AACIV+GseRmKdVFTBEAoEewAYky0NCmJlnwAABDvwtnJcrDWxUcTAKFGsAOIUdEyekS05AMAAMQOPpoACDWCHQAAAAAigo8mAELFrw5Kr7vuOq/T8vLyap3HZrNp3rx5/qwWAOgUFUBU4F6EWMG5CiBe+RXsWLNmjd/z2GxUUQPgH0+9zgNAuDEqBWIF5yqAeOZXsOPVV1/1+LthGLr99tuVn59P7Q0AQVe5fe/wQd1p3wsgIjzdi3iBRDTiXAUQz/wKdvTu3dvrtKSkpDrnAYBA0L4XVke189jAvQixItBz1RU0AYBY4lewAwAAhAbVzgFEk8r3JACIJdy1AACIIgzHCCCauO5JK1f/FOmswCSbzabERJ4jiF/U7AAAIArRRAJANMktKI50FmBSw4Yp1BJEXKNmBwAAAABYTCRqCdrt1CZB9Ah6zQ7DIGIIANHIVfDgyw7iHR3AAogX4awl6KnPKSCSgh7sWLRokRwOR7CTBQAEwFUAkURVVsQ1OoD1DQEhAGZ5GuoYiKSgN2Np2bKlWrduHexkAQABcBVA6PAS8Y4OYOvmCghlZDRQenoq+wiwsFA0O6HPKUQL+uwAAABxh8K4dwSEgPhAYBNWR7ADAAAANRAQAqyNwCasjmAHwo5emgEAMC9Sz09egsKD8pF1uPq8iRUENmFVQe+gFKiN1TuGS0iw05kbAEl08Ijg8uf5Geg52KB+spxOQ40a1bfkMzuaWL18FE/oEBzxIFbKONTsQMhVjm5btbpc5QIhbR4B0A4awWb2+RmMczClXqLsdpvlntnRyKrlo3hEh+Cwulgq4xDsQEi5LobqF4LVqstRIARQGS8uCBVfn5/BPAet9syOZuxrANEulso4NGNBSLkuBtf/rY5CCoDKuCcg0jgHAQChEAvPF2p2AAAAAAAASyHYAQAAAAAALIVgBwAAQIgwnCgQftF23UVbfoB4EbQ+O7Zs2aL3339fmzdv1tGjRyVJmZmZ6t69uy655BJ17do1WKsKmlWrVmnOnDnauHGjioqKlJWVpcGDB2v8+PFKTU31K81ly5bptdde07Zt21RWVqb27dvrsssu03XXXaekpCSvyx05ckQzZszQp59+qsOHD6tRo0bq1auXbrrpJp166qn+bmJAYmVIISDSuFZgNZzTwRGs4UQr938FoHbRNoyvp/wACI+An5xFRUWaOHGiLr/8cs2dO1dr167Vzz//rJ9//llr167VnDlzdPnll2vixIkqKoqei3v+/PkaM2aMPvvsM9WrV0+dOnXSvn37NGPGDI0cOVJ5eXmm05wyZYpuv/12rVmzRunp6WrXrp1++uknPfXUU/rb3/6m0tJSj8vt3r1bl112mebPn6+jR4+qc+fOMgxDH330ka644gp98sknAW6tebE0pBAQSVwrsBrO6eAJRo/1ruPRqFH9EOQQsJ5oGyki2vIDxJOAgh1Op1MTJkzQhx9+KMMw1LRpU1166aW68cYbdeONN+rSSy9Vs2bNZBiGPvzwQ91yyy0yjMh/Idq8ebMef/xxSdIjjzyizz77TEuWLNGKFSvUrVs3/fzzz5o0aZKpND/++GPNnj1bycnJmj59uj7++GO99957ev/999WmTRutXbtWzz77bI3lDMPQHXfcod9++03nnnuuvvjiCy1evFhffPGFJkyYoLKyMt1zzz06fPhwULbdV9yYAd9wrcBqOKeDL5Ae613HY+Xqn4KYI8D6om2kiGjLDxAPAgp2vPPOO1q1apUSEhL00EMP6fPPP9fTTz+tiRMnauLEiXr66af12WefadKkSUpISNCqVav07rvvBivvfps+fbqcTqeGDh2q0aNHy2arKMi1aNFCzz77rOx2u5YvX65t27b5nOa0adMkSTfeeKMGDhzo/r1Tp0567LHHJEmvv/66u4mPyyeffKKtW7eqYcOGeuaZZ9SwYUNJUmJiou644w716tVLRUVFmj17dkDb7C9uzIBvuFZgNZzT0SW3oDjSWQAAIKYEFOx47733ZLPZdN999+maa66R3V4zObvdrquvvlr33XefDMPQO++8E8gqA1ZYWKgvv/xSkjRq1Kga0zt06KC+fftKkrKzs31KMycnxx0YGT16dI3p/fr1U/v27VVaWlqjScpHH30kSRo8eLAaN25cY1lXHl3zAQAAAACA2gUU7Ni2bZsSEhI8Bg2qGzVqlBITE7V169ZAVhmwrVu3qrS0VMnJyerRo4fHec466yxJ0saNG31Kc8OGDZKktm3bqkWLFqbSdP3ds2dPj8u5fj948KAOHTrkU34AAAAAAIhnAQU7CgsL1aBBA6WkpNQ5b0pKiho0aBDxTkp37dolScrKyvI6Okq7du2qzFuXnJycKsv5mmZpaan27dtX67KtWrVy53Pnzp0+5QcAAAAAgHgW0NCzGRkZ+u2333TkyBE1adKk1nmPHDmi/Px8NW3aNJBVBuzYsWOS5LHJiItrmmveYKaZn5/v/u348eNyOp21Lmuz2dSoUSP3/gtUs/QGkqTExAQlJNhlGJLNJveQXHa7TYZhuIe4821+77/ZKvVrl5iY4P67rnQTE31bv0td+ai+fTabrUZekpISZLfbZLPZat2WyvO7OtzNaFi/1vya3W+1/ZaQYKuRD1e/M/6k68v8no5HXcfS33Qrp1FzHb7kw17nuVr9vPTn3Pa2DZWPh6/nsa/HqLZzsK7zwfdtNrxeA/7uS3/2QzCvmdrSNXsOVt6/Lv6c29F2jdd1r/O0/mCkEdi93Pu67HZ70PLm67VY23UXjH1Z13Om+r4M5Bx0LevrOoNxjLznt+57kkvg94mq6wrGvgz2NePLc9HXe5iv+9LsOWg23UCuxVBd44E8Z6o/Fz2Vl6r/Vj2/np7tvlwzdaVbfT9Uf96EokzkaVvqLrtULasHI2+Rfi6a/c0lWOWfYKZbW9nX0/zVik1hZTMCGB7l9ttv18cff6zRo0frf//3f2ud93/+53+0YMECXXjhhZo6daq/qwzYv//9b73wwgvq2bOnXn/9dY/zfPPNNxozZowSEhK0ZcuWOtP85z//qUWLFmno0KF66qmnPM7z9ttv68EHH1S7du308ccfS5IOHDig8847T5K0YsUKtW3b1uOy5513ng4cOKCnnnpKQ4cO9WErPfNUYHA6nbLb7V4f9hXz1D6/L7+5BJJGsNJ1bZ+kSttYMV9t0wLLb+3zu0Y78GWfB39fBu94REO6kdyXkTp/zPxW1zb7uw3xeF66fvf1fPNnXwbnnIqNfRmteQvV/cdbuoGeU7G4L2vbZl+ugWBtc/V1xeK+DDTd4G9z7J6XVt3mmtdbMNLwLb++PtO4FkNx/kT2vIyUgJqxXH311TIMQwsWLNC9996r3bt315hn9+7duueee7RgwQLZbDZdffXVgawyYPXq1ZMklZWVeZ2ntLS0yrzBTLNyk5/K6bum+7qsP5xOQ/n5xSosLJHdbtPK1T/Jbq8YXtBut8lut2nJiu9ls9l0/HiJCgpK6pzfl98KC0+406j4za7jx0vqzIcv85nJR/Xty8srUn5+sY4fL3HP9/u2F3vdlsrzu37Lzy9WQcEJj/vLl7wVFLjyYatzG7zlw59jZGa/VT8etR2jQNKte597z0d+frHy8op82peVz0t/z21P6684F4pNncdmjpG3c9CX86GubfblGvA3XTP7IZDjYTZd/87B369Fb9duIMe+8n2l+jkVqmvc1202k66Z+1Sgx7nyuoKdNzPXoqfzwey+9JZGbc8ZT/sykHOw8rK+rDPQY1RQ4DrPq5Y7fL0nBeM+4WldwdiXobhm6nouhuJaNHMOmk030GsxnOmaOd9rKy95KjvU9mz3p2zmLd2a9xjz53Zdz76iohMezx/XP09lcE/Xomu+YJTXouG5GIxrJhrS9Xacvc1fWOT9PTfUAmrG0qdPH11//fWaN2+eli5dqqVLl6pVq1Zq3ry5JOnQoUM6ePCge/4xY8aod+/egeU4QL40UfGlWUpljRo18jlN17ySlJaWJru9IiLmbVnDMNzNVyov668TJ8rdVY9cw9hVHl7Q9f+yMofKyyua2NQ2vy+/lZc7PKZfWxq+zmcmH962z1W5yZd1etqW8vLf05Jq7i/f9tHvy9e1Dd7yYWY/mMlb5f9X3je17a9A0jV7/vhyTOtK199z29v6XeeAr+exmWPk7Rys/ps/2+xvfoOdbiDHw2y6/p6DLt6u3UCOvWudtd1Xgn2N+7rN5u5rvt+nzOStrnUFO2/mrsWa54PZfektDV/Ph+rL+nMOhros4GmbPW1fqMoCvt4LapsWimvRn3t5IPcwM/sy0HtSMPIWqvtPKJ49wSi7+PNs9S3dmvcY88/92p99runezh8XT+U1z9sQ2P2n8m+RfC6a+S0YaYT6fln9OHub3+GseezDJaBghyQ98MADatu2rV588UUdO3ZM+/fv1/79+6vMk56erttuuy3itTqkiqFlJWn//v0qKyvz2Enpnj17qsxbl44dO0qSx5ottaWZnJysrKws/fLLL9qzZ4/OPPPMGssdOHDAXWPEtR4AAAAAAOBdwMEOSbrmmmt0xRVX6KuvvtLmzZt15MgRSVKTJk102mmn6eyzz/a5SUionXrqqUpKSlJpaak2bdrkHhK2svXr10uSzjjjDJ/SPP300yVJv/zyiw4dOuRx+FlvaZ5xxhn65ZdftG7dOg0bNqzGcuvWrZMktWzZUi1btvQpPwAAAAAAxLOgBDukiv4nBgwYoAEDBgQryZBIS0vTOeeco08//VQLFy6sEezIycnRqlWrJEmDBw/2Kc2OHTvq5JNP1vbt27VgwQLdfvvtVaZ/88032r17t5KSkjRw4MAq0y688EItXbpU2dnZuvfee2s0nVm4cKGpvFiZqxdwANbENR5elXvrBwAAsJqAOih955139NFHH/k8//Lly/XOO+8EssqgmDBhgmw2m959910tWLDA3V7s8OHDuvvuu+V0OjVo0CB16dKlynKuYE52dnaNNG+99VZJ0syZM7Vy5Ur37zt37tRDDz0kSbrqqquUmZlZZblBgwbplFNOUUFBge655x4VFBRIkhwOh6ZOnaq1a9eqfv36Gjt2bPB2QIxxOg05HE4N6NM50lmBxfHyFzlc44HzdP56CiCVnCiX02lo+KDucjgi147WxeFwRkU+woF7jPVwTIHw4aMIzAqoZsf999+vZs2a6aKLLvJp/ieffFIHDx702FwjnHr06KH7779fTz75pB5++GHNmDFDGRkZ2rFjh0pLS9WxY0c9+uijNZbbt2+fJKmoqKjGtAsvvNDdWevNN9+sdu3aKTU1VT/99JMcDofOOussTZw4scZydrtdU6dO1dVXX60vvvhC/fv3V8eOHXXw4EEdOXJESUlJevrppz02jYkXTqehvLwiJSUlqFEjbnIIvuovfwkJAcWB4YeCgoqe1rnGzfN0/rp+8xRAKiwuld1uU35+sZxOQ+npqRHI9e8KCkokKeL5CCVX0J57jHXw3ADChw+fVRH08V3Ad2ZXrYhQzR8qY8aM0Zw5c9S/f38VFxdrx44dysrK0t///nctWrSoRg0MX/zzn//U888/r969eys3N1c5OTnq1KmT7rnnHs2bN89rvyUdO3bUe++9p2uuuUYZGRnavn27pIoAysKFC/WXv/wloG21AtdNDqER71+mKr/8uV68EF6GwTXuL0/nr+u348e9n88OhzMqnsmGYURFPkLJFbTPzS3kHmMRPDeA8HHdQ/PziyOdlYgj6GNO0Prs8EVhYaHH0U8ipV+/furXr5/P8//44491znPRRRf5XNOlsqZNm2rSpEmaNGmS6WUBf/FlqipethHLPJ2/Tqe1gwixxOk05HQa7iE7YQ08N4Dw4MNnBWrCmhO2J+53332nY8eOxXVzDCDa8GUKABAs8V5LEABCjZqw5piq2bFkyRItWbKkym/Hjh3Tdddd53UZwzBUUFCgHTt2yGazmapJASA8uGl6R+EdAGpHLUEAQDQyFezYt2+f1qxZU+W3srKyGr9507FjR/eoJQAQzSi8V0XQB4A30dbpLYDQoCyAWGMq2NG7d+8qwYpp06YpNTW11mFRbTab0tLSdPLJJ6t3795KSEjwP7cAECYU3isQ9EGsisdCuauWXqSuU2oJItii8TqOx5EwqpcFnE5Ddrst0tkC6mQ62NG7d2/3365gB7U1AFhVvBfeCfog1sRzgC4ehvGNBdH4gh5rovU6jteRMCqXBcrKHAQ7EDMCGo3lk08+oaYGgJhCIdQ/8R70QeyI5wCd1YfwjXbR+oIei6L1Oo73kTBctToQXPFYWyhcAgp2tG7dOlj5ABDFrBAgoAomEF8I0CHcovUFPZaZvY5DXV6pGAmDl30ET7zWFgqXgIIdiG1EEeELh8NpiQABVTABAOFAoC38+KCBWBXvtYVCjWBHHHI6DaKI8FlBQYkMw5DTaVii8OBrFUwr1GYBzOK8BxCL+KCBWEVtodCiMWEccjoN5eUVKT+/ONJZCbtYK8hHQ+0bwzBUXh6bbTT9Od6uYGDlr0PhEmvnZ6ixP8LH01dRxDauH8Qj7l8AKqNmR5xyvdCFkq8FrXC90Nf1AhttBcNg176Jtu0LpeoBCzOdxLmCgXa7LWy1WejUrir2R/jxVdQ6PAVsg3ksoyEIDwCALwh2IOh8bTcZ7uY01ZtjVM9HtLXzDFYbvnh8cawcsLDZbKY7iat+joQandpVxf6IHL6Kxr5QBWxpAgsAiDUEOxB0vn4hdBXIkpISwtIpj6s5hrd8hPNLvi+C1YYvXl8cXccyMTG0wZ1g1pihU7uq2B+Af0IRsA33MxsAgEAR7EDI+PKFMBzNaXwR7i/5kRAN+9lqItW3BwBEQrQ8swEA8AXBDkQlK/UvQftm6/LWNAoAwoHnCwAA3hHsQNSxytdyV18ZtbVvtlJQJ5b5exy8NY0CPOHFNDzi4b5K/xkAANQtbD0V5ubmqkuXLuratWu4VokYVVBQotzcQuXlFcV0sMPVV8bx4yU1pgVzmMd4KNiHSiSHmUX84MU0POLpeo7nIeQBAPBV2Gt2GIZ1Cx8IjkC+lkfji7+nAncwhnkMZHhVVPDUOS0QbPHSsWMwa674cy+P1s6mQ4X+MwAAqB3NWGAJsTq8aiBfHwMdXhUVCHLErlhqFmKVF1Nv+zxYNVcCvZfXdT1HY0AcCLZYujcCQCjFxhshUIfKNSUKCmo2G7Eqp7OiFgw1phBvaBYSfrXt84KCkqA0qQjVvTyYTQeBaEWTOdSGYC/ikamaHQMHDvR7RbyMIRys8OU0GHigWQ/HtKqCghLZ7TZLNwuJNrXtc8Mw5HAE7zkf7Ht5MJoOAtEuXprMBUM8PVM99WfE/c+zeDov4oWpYMe+fftks9kIXKAGbg7RIVab88SKSFQN9vRF2gqFlED3ZbBfrlE3K+zzYNTq4HmHaGaVJnOhEuvlJPozCo1gnBfBKCPyfAk+U8GOpKQklZeXa+jQoWrbtq2pFRUXF+uVV14xtQyiX6w/NKym8tdLp9OgH48g8bdqcDAeWlb7Ik01a0RCMK5FqwYegXgSq+WkUPdnFO8COS9cxyaQcg2DDoSOqWBH586dtXXrVp1++um66qqrTK0oNzeXYIcFxepDw+r4qhNcZqsGB6PKaPUvBFbpZ4Bq1rEnlr80BXM4WqsFHoFgiNXOUGOtnER5Ozz8OS9cx+b48RKlpaX4tV4GHQgdU8GObt26acuWLdqyZUuo8oMYFWsPjVjl2s9WeOmNNWaqBgdSZTQeaj5QzTo2RHM7b19fsEIxvLRVAo9AIOLhWRWNeHZGr0CfC65nVGIitTqCyVSwo2vXrpJEsANusfzFLxYVFJRYqqBt5fPH3xcraj6ETqx+gYyUaG3nbfYFi+rbQPDxrAIQC0wFO04//XRlZWWpqKhIhmHIZvO90FO/fn3deuutpjOI6BWtX/yszDCsU2jn/PEuUjUfIhF8Csc6g9GeNl5FY6Ag2CPxWDnoCoQStfTiB/dJxCrTNTtWrlzp14pSUlIIdlhMQUGJ++U7GC+r3EjjS7DPH3jny7UV7uBTOJtIpNZLCrg9bahx//NdsEaFoUM4WAH3juBjn1YVrH6PfMX+RzCZCnYAlRmGofLywCP6wexALpoFcvO24o0/WOePr6y4D+tiZvSIcAcvw9VEovK9JZznm68Y0Spy6BAOsSxeyk7hxP3Ys+rlg1DinEawEexAxIWiA7loEsjXQwozgbPyPqwroGBm9IhgBZ/MFBbDcb1XLqRFY+0hetiPLDqEQ6yyetkpErgfexbOj1PhDKwgPoQk2HHo0CE5HA5lZWWFInlYUCze1HytKWAY/n89pDATuGjbh8HoJNPXJiCuczScQZ5oKyxWLqSZDXaEszYQ7d4BmBUNzzQriuX7cbTVYjWbn3DX+oX1hSTYcfnll+vo0aOM2gJL8qemQCBfD61SmInkSBjRsA+DOUyfL01AIl2bJZYLi2aa/wBAtIq2F1+ETrTVYo22/CB+hawZi2FwUsOaoq2mQCwI1kt+LAv2MH11nXtUBfWfmeY/ABBtwtkBNKJDtJVNK9dq9iU/BOYQKvTZAZ9F8st8tAn0QRJvN/VgDxUZq8I5TB9VQQPH1yggfljpuRyuDqArs9L+i1XREOSozJf8UAMEoUawA3UKZvX7eBevN/VgDRVJYQqwDrPXM9c/QsWKtSDC9eLLEM4IRLTVSIH1EOxAnYJd/T6ecVP3D1Vyo0skanlRs8w6zPaJEok+VAisxJdgD70dTxjCGYGiPIxQCkmwg/46rCec1e+tLtpv6tH4UhmJKrmoKRK1vKhZFp0CuU+Y7RMlnH2omBk6GdZBs7/AMITz7/dEAqWIZdH4DhCokAQ7xo0bp6KiolAkDSBEov2lMtqDRPEgErW8qFkWXYJ5nzDblC8cTf+ibehkANHPdU90OqmBitgVze8AgQhJsGPs2LGhSBZACPFSCV9EopYXNcuiR7zcJzjfAPjK1QzKhRqoiEVWHUyAPjsAuAX7pZLqnID1EHwCgN/RDApWEKzBBKKNz43riouLQ5KBUKULxKNoCS546lAQQPhEy70AsYNzJrSs2BYeQPTgHuOZz8GOgQMHas6cOTpx4kRQVvz999/rpptu0uzZs4OSHqIHBabwq21I20gcj8rt3vPyigh2AGESr8NbR1osP/eqB6cRXK79a8W28ACig1X72wgGn5uxlJWV6amnntKsWbM0fPhwXXrppTrllFNMraywsFAff/yx3nnnHa1evVqGYejss882nWlEJwrZkeNtSNtID9fKeYBgi+WXSpdQbkP1ewFtxkPLCsNi0ylraLn27/HjJUpLS4l0dgBYkFX72wgGn4Mdy5cv17Rp07RgwQK98soreuWVV3TSSSfprLPOUo8ePdSlSxdlZmaqcePGSk5O1rFjx3Ts2DHt3btXmzZt0qZNm7R27VqVlFR04nPSSSfp3nvv1Z///OdQbh/CyNsLN8Kj+j5nuFZYiRVeKqtvQ6iGNa18L4i1fRRrrHSfpVZHaFEmAhAqVu1vIxh8DnZkZGRo0qRJuvbaazV79mwtXbpUP/30k3bs2KEFCxbUubyrl+IePXroqquu0mWXXSa7PX7H47YqghzRJVLHwwpf3xFdrPBSWXkbbDYbX9AtguceAFhPpPrAqNwJeKg+isQT06OxdOjQQY888oj+8Y9/6MMPP9SXX36pdevW6ejRo55XkJiorl27qnfv3rrkkkvUpUuXgDMNmMGLd/jQMSlCyQovla5tSEykAIPoxXMTCB6up9jiCjZEqg8M14eRhAQ7H0WCwO+hZxs0aKArrrhCV1xxhSRp7969Onz4sHJzc3XixAmlp6crMzNT7du3V2oqBwrhx4t3+FVu+11W5mCfxwAKYQBcrNj3FiMUIFKs0PwyHrmCDUlJCRHrA6PiXKl6/6W85h+/gx3VtW3bVm3btg1WckDAYvXF2wo3M6sUkq2MYCCA6qzW9xYjFMSPaCw7WaH5Zbyq3JQk0jyV1ziPfBe0YAesIdQPi2CkbzaNWHmRi8QXtWgsHCA8YjUYiJq4jhFMVghyuDBCQXyI5tpIVrqefMHzKPg8ldcIdviOYAfcQvmwCMaLfLhGMoiUcH5R46s+XDj+sS2aC/lAuFVvssIIBdEnFM2KCgoqRnqMt8BCtOF5FFrsV//EbbBjy5Ytevnll7V27Vrl5+erefPmOv/88zVhwgRlZmb6ne6qVas0Z84cbdy4UUVFRcrKytLgwYM1fvx4r32XnHLKKbWm2bRpU3311Vd+58lXoXxYBONFPtZGMvAnuh2uBzVf9QFroJAPVKDJSvQL1TEyDEPl5dHR5CCe8TxCNIrLYMfy5ct19913q6ysTE2aNFHnzp21a9cuzZ8/X9nZ2XrzzTf96n9k/vz5mjx5sgzDUMuWLdWqVSvt2LFDM2bM0PLly/XGG28oPT3d6/KnnXaakpOTa/xe2zLBFOqHRTBufrEwkkEsdfAW7PxRfTHyOAbxJR4K+ZzT8EU0NFnhXK1dNBwjhE44n0fxdK3F07aGQtwFOw4dOqT77rtPZWVlmjBhgm655RYlJiaqoKBAd911l7788kvdeeedevvtt2Wz+d4eavPmzXr88cclSY888ohGjRolm82mQ4cO6eabb9YPP/ygSZMm6cUXX/SaxtSpU9WmTZuAtxHB4091S6t18OaLWArwWBXHAFbDOQ0zItlkhVE3fBPtzYp4qYx+Vm/SXhnPwOCw7hnixaxZs1RcXKxevXrpjjvuUGJiRbynYcOGeuaZZ9SwYUNt3rxZn376qal0p0+fLqfTqaFDh2r06NHuQEmLFi307LPPym63a/ny5dq2bVvQtwnBF+gY205nRXQ7lm9MZh76rgBPbm6h8vKKYnq7YxXHAFYTL+c0Q6PGPjPnauVnK8c+OtCPWeyofK0VFJREOjsBqaucHUvPwGgOFMZdsGPZsmWSpFGjRtWY1rhxYw0ePFiS9NFHH/mcZmFhob788kuv6Xbo0EF9+/aVJGVnZ5vOM8LPdYPJzy+OdFbCzt9IshUCPLGOYxA/orlgEUxWPqddL1j0M+FdLJ3ndZ2r1Z+t9DESPSr3YxbtL5X4/VozjNg8TmaCa9H+DKx+X4tGcdWM5cCBAzp06JAkqVevXh7n6dmzp9566y1t3LjR53S3bt2q0tJSJScnq0ePHh7nOeuss/T111/Xmu706dN1+PBhORwOtWjRQn379tXFF1/ssR8PhF40jbEdTvHYDAeIFcH4AhlLL5DRJNj7zfWCdfx4idLSUoKadjQIpNaCFZuFGEbVZ6skJSUlqFGj+lyTUSLWanVw3sQmKw0SUPm+Fq2DR8RVsCMnJ0eSlJSUpJYtW3qcx9Ux6d69e1VWVqakpKQ60921a5ckKSsry+v87dq1qzKvJ4sWLary95IlS/TCCy/oxRdfVLdu3erMhy8qd+xZvZ1bXe3ezM4fSt7W7en3QPIZTdscDGa3x2631ShgupZxPWR93Sd1rdvMMTWzfDAFY521LVN5Wqi2z9/9bjZv/lyLgd6T6prH131vZll/82j2enD9XfkFubzc6fEarS0dh/P3F0ibzfv1XRd/zwFv0yP9PKotveoBJpvNVmsn2f7e62qbHuh90Oy14u98CQl2r81AzV5/rs4sXYGOYO63uuYLRhnDc5oVH1Bc1131oI6na9LXfIRiGbPLh+IYBbLecN3zzezbYDxbPZ03ZjvuN3NfiqZyWKjXF8hzyZ+AbF3PcG9Cse/9v69Uva9Fm6AGOwzDUG5urkpKSpSVlRXMpIMiLy9PUkVzFW+dj7pGPnE6nTp+/LgyMjLqTPfYsWPudL1xTXPNW9nAgQM1dOhQdenSRS1btlRhYaG++eYbPffcc9q7d6/Gjh2rd955R61ataozL7Wx223KyPAeBTbbO3Y09qbtKU/BzGc0bnMg/N0eV8Hf6TT8TsPX5YI9XzAFa52uwFFt6YVq+4Kxf3297iJxjwlk+3xZtq55/F1/Xcv5WxMgwV5RmElIsHv8AhPM6zmQfRPp51Hl9FwBJsn7fgskb9WPZSDXTiD7IdTXW6jvb+HYb6Fch5lzK1afd8G+T5iZJxh5CWQ5M+nVViao/ps/9yRf8+HLNH/mi2aB7Aezz+Voez/xlkZttfRi4ZgHJdjxww8/aMaMGfr6669VXFwsm82mLVu2uKcfO3ZMzzzzjCTpn//8p1JSIlNd88SJE5JUa22Nyk1GXPMHM11PaU6fPr3K3/Xq1dOQIUPUr18/XX755dq/f7+mTZumyZMn+5Qfb5xOQ/n5Re6/ExLsVU7S/PziWpttmJ0/lGw2mxo2THHnq3KeJAUtn9G0zcEQrO1xfX1yOg2f20zWte7q0+vKYySOTTDWWT2Nyl/aXWPUB2tdvqzfn/3raZpU87rz9Fsw7zHezpnKast39flqy29t52f15bytv7a8V0+nruvD1/PB7HUXSLqV8+/tt8rqOqfC+TzydXtCsc+rN2Mxs25/r2dP/N0+b2n4sk/NrLuubfDnHPS2fjPXe7CfUcE8VmbTDXSdoThGvhxDf877YN7zzRyP2s4LV9m2cpmg+lC9oSwLSOauz0iXkQM9pz3x9x4m1byXm1mXWWbz5mt+qqfhcNbet5CvZaxICjjY8c477+ihhx5SeXm513kaN26sPXv2aPXq1erTp4+GDBliej2TJ0/Wq6++anq53r17a/78+ZIqggiSVFZW5nX+0tJS9/9d89fFTLq+pilJmZmZGj9+vP73f/9XK1as0GOPPWZqOFxPahv/2uFwmhof2+z8wZaXV1Qjou3pphHMfEZ6m4Mtktvj67qDPV8wBWOdBfkVAY66+kcJ1fYFY/96u+78XZe/8weShr/5raug4u/661rO333jbTnXl8RA0vXlN1/y4k8+gn19BPNZUtc+r37dB7LuQPZDqK+32s4Hf9Zdvb8Cf85Bs+uP1LEJZVrhXKe/9wlf0jWbt2De84N53VXvM61685RQlgV8zWOg80Uzf+9hUs17eSDrMisYQSZPaRTk1wy4VV8m2o95QA18duzYoUmTJqm8vFzXXnutFi1a5LXZx7Bhw2QYhr744gu/1pWamqr09HTT/9LS0txpVG5K4u1rtKupi91ur7JsbWprouLiS1MXT/74xz+68+XKGyqYqVUARCvDiO6etmFt/o6+BP+xzwNTvQ8VfzDkK6JVtI++gfhiGLE/WENANTvmzJmjsrIyXX311XrwwQclSQkJCR7n7devn6SKJi/+uOuuu3TXXXf5l9H/X4cOHSRV1MA4cOCAx35F9u7dK0lq06aNT52TVk53//79Xjs13bNnT5V5fVU5LYfDYWpZAAgUvb1bWyRGX4r3c6r6Po/WTt3qEqnjWHkkA6fTMN1fgafOUwEgXln9mRxQzY7Vq1fLZrPpxhtvrHPeFi1aKCUlRQcOHAhklQHJyspS8+bNJUnr1q3zOI/r9zPOOMPndE899VQlJSWptLRUmzZt8jjP+vXrTacrST/99JOkiuYvrs5TAcS+aH+48PU5fgTyJdHsecw5VSGWv95WvzdEisPh9Kt2Z0FBibttOgDEq2AMZR8LAgp2HD58WPXr1/c6jGt1KSkpPnf6GSoXXnihJGnhwoU1ph07dkzZ2dmSpMGDB/ucZlpams455xyv6ebk5GjVqlWm0y0vL9ecOXMkSX379lViYlyNFAxYUqwEEVxfn3NzC5WXVxS1+Ywm0R7ACiZ/z+OCghLOqRhX+d5QUFAS6eyYZoVq2QAQqMq15Kz8TA4o2JGcnKyysjKfIuulpaUqKChQw4YNA1llwMaNG6eUlBStXbtWU6dOdTcNKSgo0MSJE1VQUKCuXbtqwIABNZa98sorNWDAAM2dO7fGtAkTJshms+ndd9/VggUL3Pvk8OHDuvvuu+V0OjVo0CB16dKlynL/+te/tGTJEh0/frzK7wcOHNDtt9+uDRs2KDExUbfcckuQ9gCASIqlIEIsf30Op2j50h1O/p7H9FFjDa57A/1mAYB3sdA/UDR/eAuGgKoKtG3bVtu2bdOuXbv0hz/8odZ5v/zySzkcDp100kmBrDJgrVq10pQpUzRx4kRNnz5dCxYsUMuWLbVr1y4VFRWpadOmev755z2OenLo0CHt27dPBQUFNab16NFD999/v5588kk9/PDDmjFjhjIyMrRjxw6VlpaqY8eOevTRR2sst3PnTs2cOVMPPvig2rZtq8aNG6ugoEC7du2SYRiqV6+eHnvsMZ1++ukh2R8Awi9cfSNEC6vXeKjcB4PNZjPdh0CsirfzGEBkxcKLI+ASr/0DRdt1GlDNjv79+8swDM2bN6/W+Y4fP65nnnlGNptNAwcODGSVQTF48GAtXLjQ3aRl+/btysjI0DXXXKP33ntP7du39yvdMWPGaM6cOerfv7+Ki4u1Y8cOZWVl6e9//7sWLVqkzMzMGstceeWVGj16tLp06aLCwkJt2bJFBw8eVOfOnXX99ddr6dKluuyyywLaXgCIhFhpshMMfOkGYo+vgVirB2yjnetZEo8vjohd8dY/ULRepwHV7Lj++uv1xhtvaOHChcrIyNDYsWOrTC8pKdEXX3yh5557Trt27VKzZs00atSogDIcLN26ddMLL7xgapmVK1fWOU+/fv3cI8/44txzz9W5555rKh8AEAsiMdIHANTFTCC2+nyxOnqOv6Ih0ON6liQlJahRo+j6agx4U9E/UPyUe6L1Og0o2JGZmampU6dqwoQJeumllzRr1iz3V61zzjlHeXl5cjgcMgxDqampeuGFF5SaGh/VewEANHUAEH3MBGILCkpkGIZ7vngJdkTbSA2uABWA6BWN12lAzVgk6U9/+pMWLFig3r17q7y83B3c+O2331ReXi7DMNS7d28tWLBAf/zjH4ORZwBwi4avTgCA6OHLc8HXDpjjtVPdeBmpAYC1BWUs01NOOUXz5s3Tvn379O233+rw4cNyOBxq1qyZzjzzTL/7wAAAb+KpPwgAsJJQdWDn6bkQLzUxQoXnK4BYFlCwY9q0aZKkyy+/XK1atVLr1q3VunXroGQMAGpDfxAAEFtC3YGdp+cCwQ4AiF8BBTv+/e9/KyEhQTfddFOw8oMYQLMBRAuCHIh13E8RT6p3YBeK85/nQmzg3gcgHAIKdmRkZMjhcCgpKSlY+UEUo9kAgoECDkB1e8Qvp9NQWZmD8kSc4t4HIJwCCnZ06dJFq1atUm5urjIyMoKVJ0Qpmg0gEATLgN9R3R7xjPJE/OLeByCcAhqNZfTo0XI6nZo7d26QsoNo52vv5UB1rgJObm4hPbsD4n6K+Mb5H7849rGBmriwgoBqdlx44YX629/+ppdfflllZWW64YYblJmZGay8AbAYvuABAKzA6i+CVt8+eFdyolxOp0FNXFhCQMGO6667TpJUv359zZkzR/PmzVO7du3UpEkT2e2eK43YbDbNmzcvkNUCAAAAYRcPTTKtvn2oXWFxqex2m/Lzi1VW5uAcQEwLKNixZs2aKn87HA7t2rVLu3bt8rqMzUa7PAB146sSgHjHfTD6xEN/IwUFJTIMw7LbB99YIdjFPRQBBTtuvfXWYOUDACRRfRLmUZiB1cRD7YFYZvUggGFU9KmB2vHsiV7Vy5IJCQF1U2kZDkfFdV3X/rDSuU2wA0BUofokfFX9hZDCDKwi1moPWKlgHGzsG+shGBn9KpclnU5D6empkc5SVCgoKJEkr/vDikNDBxTsAIBQoQCBulR+IbTZbFFVmOEFB4GKhSAHL33esW+sK9aCkfHMVZMh2KL5GV9b3gyj9nPVikNDE+wAAMQs18M4MTE6anVE6wtONBfMELt46fOOffM71wunlfZBvB/TeBXNNUrN5s1bucBq53bQgh1btmzR+++/r82bN+vo0aOSpMzMTHXv3l2XXHKJunbtGqxVAQAQlaLtBSdagy+wjmg4z6MV+6ZCQUFJzN5/MhrWj3QWEEWiuUZpbXmrHNiIt3JBwMGOoqIiTZo0SR9++KGkqtVjfv75Z61bt05z5szRxRdfrEcffVSpqdFzUgAAEGzR9IITbcEXxB9qFcE1sksscb0QDujTOdJZQZSJthqllVXPm6dO/+OtXBBQsMPpdGrChAlavXq1DMNQs2bN1LdvX7Vs2VKSdPDgQa1evVqHDx/Whx9+qKNHj2r27NkMPwsAQJjEQ2EG0Sfevh4idnkKyLleCJOSEtSoEbU7EJu8dfofT+WCgIId77zzjlatWqXExETdf//9uuqqq2S3V41yOZ1Ovfnmm3riiSe0atUqvfvuuxo2bFggqwWAiOJLZWyrfPw4lkBoxNvXQ8SeugJyrulArIvngHNA9W/ee+892Ww23XfffbrmmmtqBDokyW636+qrr9Z9990nwzD0zjvvBLJKAEHAC55/+FIZ26ofP44lEFpOp6Hycq4vRCdXQC43t1B5eUVRfZ5SbgP8E1CwY9u2bUpISNCoUaPqnHfUqFFKTEzU1q1bA1klgADwsh6YWCoYoabqx49jCcQ21zON5xn8Fe0BOcptQGACasZSWFioBg0aKCUlpc55U1JS1KBBAxUVFQWySgABoFpx4Nhvsa368YumY1m5ynQ05QuIVq5nmuv/gNVQboMVJHho/REuAa05IyNDBQUFOnLkSJ3zHjlyRPn5+UpPTw9klQACFO1fMYB45SrUUtME8K56dX5eAGF1lNtCg6ZB4eFwONUgNTli6w8o2HHGGWfIMAy9+OKLdc77wgsvyDAMnXnmmYGsEgAAy+LFDfCM6vwAgqH6cKwIrYKCkoju54CCHVdffbUMw9CCBQt07733avfu3TXm2b17t+655x4tWLBANptNV199dSCrBHxGxBYAAGugzyQAwVB5ONaCgpJIZ8fyDCOy9+qA+uzo06ePrr/+es2bN09Lly7V0qVL1apVKzVv3lySdOjQIR08eNA9/5gxY9S7d+/AcgzUga8/QHgQUAQQTtR8AhAs1OqIDwEFOyTpgQceUNu2bfXiiy/q2LFj2r9/v/bv319lnvT0dN12223U6kBY0JkTEFqeAop2uy3S2QIAAADcAg52SNI111yjK664Ql999ZU2b97s7rC0SZMmOu2003T22WerXr16wVgV4BOCHEDoeAooEuwAAABANAlKsEOS6tWrpwEDBmjAgAHBShIAEKUIKALBRbMwAACCK3KD3gIAAATACgEC+pkCACA0AqrZceTIEX3wwQfKzMzUJZdcUuu87733nvLy8nTJJZcoMzMzkNUCAIA4ZqUAAf1MIZysECAEAF8FVLPjvffe0xNPPOFxyNnqtm3bpieeeEJLly4NZJUAAEScw+GM+ZfsUAnHy5TVhiF1Og2Vl3M+IXSqBwiBYCGAhmgWULBj5cqVkqTBgwfXOe+wYcNkGIY++eSTQFYJALC4YBScQl34KigoscRLdjCFu7YFAQLAd5UDhAUFJZHODiwgWmrYZTSsH5H1IjYE1Ixlz549Sk5OVqdOneqc9+STT1a9evW0d+/eQFYJALCoYBScSk6Uy+k0Ql74MgyaG1THKD1AdHNdl4mJdNmHwEW6CZ6rzDCgT+ewrhexJeA+O9LS0nyev379+vrtt98CWSUAwKKCUXAqLC6V3W5Tfn6xysocBCTCjD4ngNjBF3EEKpL3fFeZISkpQY0acS7Ds4BCu2lpaSooKNCJEyfqnPfEiRMqKChQ/fqcjADgq3hrCxuspgn0pwEAnvFFHFbhOpcBbwIKdnTu3FlOp1OffvppnfOuXLlSDodDHTt2DGSVCIF4e5kCYkG0tIUFAFiL64t4fn5xSNKnxgiAaBFQsGPAgAEyDENPPfWUDh065HW+Q4cO6amnnpLNZtOgQYMCWSWCiJcpIHpZbbQJAED0CMUXcWqMAJ7xYTlyAgp2/PWvf1XLli114MABDRs2THPnzlVOTo5KS0tVWlqqnJwczZkzR8OGDdOBAwfUokULXXXVVcHKOwLEyxQQ3RhtAgAQK0JdYwSINQz5HHkBdVBav359/fvf/9YNN9yg3NxcTZkyRVOmTKkxn2EYysjI0IwZM5SamhrIKhFkdCYHAACAYKAPBeB3lTtet9lsSk/nPTjcAh57qlu3blqyZIkuvfRSJSQkyDCMKv8SExM1bNgwvfPOOzr11FODkWcAAAAAAKKaq5auYfBxORICqtnh0rJlSz399NN65JFHtHnzZv3666+y2Wxq1qyZTjvtNKWkpARjNQAAAAAAAHUKSrDDpX79+urVq1cwkwQAAAAAADAl4GYsAAAAAAAA0SSoNTtcPv/8c7311lvatWuXkpOT1bVrV1133XU65ZRTQrE6AAAAAAAAN1PBjpycHD388MNKSkrSjBkzlJycXGOeF198UdOnT5ckd0cs27Zt07vvvqvnnntOf/nLX4KQbQAAAAAAAM9MNWNZtWqV1qxZo4YNG3oMdKxbt07//ve/3UGO9u3bq2vXrrLZbCovL9cDDzygo0ePBifnAAAAAAAAHpgKdqxbt042m00XXHCBx+kzZ86UJKWmpmrOnDlatmyZFi9erMWLFysjI0OFhYV6++23A881AAAAEEeapTeIdBYAIKaYCnb8/PPPkqSePXvWmFZcXKyvvvpKNptNY8eOVd++fd3TunTpoptuukmGYeirr74KMMtATRQAAACAFZWcKJfTaWj4oO5yOJxyOo1IZwkAYoKpPjt+++03paSkqHnz5jWmbdq0SeXl5bLZbBoyZEiN6ZdccomefPJJd8Ak0rZs2aKXX35Za9euVX5+vpo3b67zzz9fEyZMUGZmpun0fv31V3311VfavHmzvv/+e23dulUnTpxQ7969NX/+/DqXLysr07x58/Tee+9pz549SkpKUpcuXXTttdd6rUkDyek05HA4KQAAAABLKiwuld1uU35+scrKHJR1AMBHpoIdubm5atDA8xf0zZs3S5IaN26sjh071pjetGlTJScnKz8/349sBtfy5ct19913q6ysTE2aNFHnzp21a9cuzZ8/X9nZ2XrzzTfVtm1bU2l+8MEHeuKJJ/zKz4kTJ/S3v/1N69evV0JCgk466SQVFxdrzZo1WrNmjW688Ubdc889fqVtdU6noby8ItntNjmdBgUAAABgSXzUARApsVqL3lQzlpSUFOXn56u0tLTGNFew49RTT/W6fP369eV0Ok1mMbgOHTqk++67T2VlZZowYYK++OILLV68WF988YXOPfdc/frrr7rzzjvdnaz6Ki0tTX/605900003adq0aZowYYLPyz799NNav3692rRpo6VLl+q9997Txx9/rOnTpys5OVkzZ87UypUrzW5q3HA6DZWXUwAAAAAAgGCpXos+1pgKdrRu3VqStH79+iq/G4ahtWvXymazqUePHh6XLSsrU0FBgRo3buxnVoNj1qxZKi4uVq9evXTHHXcoMbGickvDhg31zDPPqGHDhtq8ebM+/fRTU+mOHDlSc+bM0d13362//OUvatKkiU/L/fbbb/rPf/4jSZo8ebL+8Ic/uKcNHDhQN9xwgyRp2rRppvIDAAAAAIC/XLXoc3MLVVBQEunsmGYq2NGrVy8ZhqEZM2ZUqaHxwQcf6LfffpMknXvuuR6X3bZtm5xOp+nmIcG2bNkySdKoUaNqTGvcuLEGDx4sSfroo4/Ckp+VK1eqrKxMHTp0qNKpq8tf//pXSdIPP/ygPXv2hCVPAADAumK1OjIAIPxctejNtnyIBqaCHVdeeaUSExO1du1aDRs2TM8884zuvfde3X///bLZbOrYsaPHkVok6csvv5QkdevWLfBc++nAgQM6dOiQpIrAjSeu/G/cuDEsedqwYYMk6ayzzvI4vUWLFmrTpk2VeQEAAMyqPqoHAABWZirY0alTJ3d/Ftu3b9esWbO0dOlSlZeXKyEhQQ8//LDXZd977z3ZbDb16dMn4Ez7KycnR5KUlJSkli1bepzHVfNk7969KisrC1ue2rVr53Ue17Rdu3aFPD8AAMCaKo/qEYvVkeMVNXEQTpxvsBJTo7FI0g033KB27dppzpw52rZtmySpe/fuuu2227zWlvjqq6+Um5urpk2b6pxzzgksxwHIy8uTVNFcxWazeZwnPT1dkuR0OnX8+HFlZGSENE/Hjh1z58kb17RgjGSTmGgqvhX3EhLstf6N4GFfh0eo9nOkj18w1l9bGlbYvkjwNd+R3j5f1u9rnsK5Lf7uX0my2211zmMF/hyP2q79UKu+Pofz944BbTZbUMtxZs6XQNKtK73apvtyLPzZjkhdz5G+19WWj+odUQb7fAsVf49zXff5uo5V9b/ruqcG89ibTTucZb1oOcddTAc7JOmCCy7QBRdc4PP8Z599tlavXu3PqoLqxIkTkipqdniTnJxcY/5oyVNJSWBfYex2mzIyiNYGolGj+pHOQtxgX4dHqPZzpI9fMNZfWxpW2L5I8DXfkd4+T+v3N0/h3JZA9m+k93m4mN3OSO+XBHvFi0JCgl3p6alhWWewtzmQ9HxZ1p/0o+V6jvT55VI9H+E830LJ7HM8kOd+WlqKqfmDeezDuS6z6Ub6HPcr2BEJkydP1quvvmp6ud69e2v+/PmSpHr16klSrc1TKg+r65o/lMzkKSUlxes8vnA6DeXnFwWURrxJSLBXuUjz84tp5xwi7OvwCNV+jvTxC8b6a0vDCtsXCb7mO9Lb52n9kvzKUzi3JZD9K/m3fbHGn+NReZloOBfDdf4Ea51mt8FbPqovaza/vqZbm2Afj0jf66ItH8FQ13GWPL9w13UfrGsfVZ9+/HhJlYBHXfMHss/N5i2cZT1P8zRoUC9iNTxiJtiRmprqbmJiRlpamvv/ruYgx44dk2EYHpuyuJq62O32KsuGSqNGjdx58sY1zTVvIMrLY/NGFi0cDif7MEzY1+ERqv0c6eMXjPXXloYVti8SfM13pLfPU6HQ3zyFc1vM7F9/l411Zrcz0vslEusP9joDSc+XZf1JP1qu50ifX9GWj2Cr7QXf7H2wrn3kdFYdqaSu+YO5z8O5LrPpRjqIFjPBjrvuukt33XVXQGl06NBBUkUtigMHDigrK6vGPHv37pUktWnTptamJcHSoUMHffvtt9q9e7fXeVxDzrryDwAAAAAAvIv+XmeCKCsrS82bN5ckrVu3zuM8rt/POOOMsOTJtZ5vv/3W4/RDhw7pl19+CWueAAAAgEAwqgeASIurYIckXXjhhZKkhQsX1ph27NgxZWdnS5IGDx4clvwMHDhQSUlJysnJ0apVq2pM/89//iNJ6tq1q9q3bx+WPAFArKJwDQCRVX1Uj+rV+4FQoywAl7gLdowbN04pKSlau3atpk6dKofDIUkqKCjQxIkTVVBQoK5du2rAgAE1lr3yyis1YMAAzZ07N2j5adq0qUaPHi1JevDBB7Vz5073tJUrV2rWrFmSpFtuuSVo6wQAqwlX4ZoCFADUzuk0lJdXpNzcQuXlFRHsQFgRaENlMdNnR7C0atVKU6ZM0cSJEzV9+nQtWLBALVu21K5du1RUVKSmTZvq+eef99h56aFDh7Rv3z4VFBTUmHbgwAENGzbM/bdrBJVvv/1Wffr0cf9+ww036MYbb6yy7L333qsffvhB3333nS655BJ17txZRUVF7r46xo4dq0GDBgVj8wHAklyFa7vdJqfTCHoBhy+VAOC7UNyHAV8UFJTIMAz3OWi313ynq44PGdYVd8EOqaKJStu2bfXSSy9p3bp12r59u5o3b64RI0ZowoQJatKkiek0HQ6HeySXysrLy6v8XlJSUmOelJQUvfrqq5o7d67ef/995eTkKCkpSb1799Y111zjbnoDAPAulIXrUAdTAAC/4+UTZrnOGcMwfB55hA8Z/oml6zMugx2S1K1bN73wwgumllm5cqXXaW3atNGPP/7od36Sk5M1fvx4jR8/3u80AAChEw1BjlgqYACAP3j5hBmBBCz4kGFOLAaHTAc7ysvL3bUT0tLSfFrm+PHjkqT69esrISHB7CoBAIhrsVjAAAB/VG+GANQm0IAF55nvYjE4ZLqD0rvvvlu9evXS/fff7/My//znP00vA0QTvqYCiCQ6/AMQL1zNELjPwVdOZ+jOGd4Bqgrlvg4FU8GOn376ScuXL1daWpoef/xxn5d79NFHlZaWpg8++EA5OTlm8whEDF9TAUSLWCtgWAUFXQCIP7wDWIOpYMf7778vSbrqqqvUqFEjn5dr3LixrrnmGjmdTr333nvmcghEEF9TASA+UdAFgPjFO4A1mAp2rFu3TjabTRdccIHpFbmWWbNmjellgUjiayoAxB8KukB4UYsK0YZ3gNhnqoPSnJwc2e12de3a1fSKTjnlFNntdu3cudP0sgAAAOEWKx2wAbGMWlQAQsVUsCM/P18NGzaUzWYzvSK73a6GDRuqoKDA9LIAAAChxpdlIPxicYQHALHBVLCjfv36Kiws9HtlRUVFSklJ8Xt5AACAYOPLMhBZ/gQ5CE4iFnHehpepPjsyMzNVXl6uPXv2mF7Rnj17VFZWpszMTNPLAogfPASA+BWp65/+OYDYUXKiXE6nQXASMYWgemSYCnacccYZkqTly5ebXtGyZcskSaeffrrpZQFYHw8BIH5Fw/VPR3RAbCgsLpXdblN+fjHBScQMguqRYSrYcd5558kwDL3yyis6fPiwz8sdOnRIs2fPls1m03nnnWc2jwDiAA8BIH5x/QMwiw8jiDUE1cPPVLDjwgsvVPv27ZWXl6dx48b51Jxl9+7duuGGG5Sbm6t27drpoosu8juzAKyNhwAQv7j+AQBAMJnqoNRut2vKlCm67rrrtGPHDl122WW67LLLNHDgQHXt2lWNGzeWJB07dkxbtmzRihUrtHTpUhUXFys5OVlPPvmkXyO5AAAAAAAA+MpUsEOq6Lfj+eef13333afjx4/rrbfe0ltvveV1fsMwlJqaqqeeekp//OMfA8osAAAAAPiKjs+B+GWqGYvLgAEDtGjRIg0ePFg2m02GYXj8Z7PZNHjwYC1evFiDBg0Kdt4BAADgI176EE+qd3wMIP6Yrtnh0r59ez3//PM6cuSIVq9erZ9++kl5eXmSpPT0dHXu3Fl9+vRRkyZNgpVXAAAAmORptBu7nWbFsDZXx8d2u002m03p6amRzhKAMPM72OHSpEkTXXzxxcHICwAAEcfXb1hN5Zc+p9Mg2IG44TrfExP9qswOIMYFHOwAAMAKPH39BqzC9dIHVEeAF4BVmQ52LF68WCtXrlTLli310EMP1Tm/YRiaPHmyDh48qAsvvFCXXnqpXxkFACCUPH39BmIBL6vwR/UAb0ICtR8AWIupYEdBQYGeeOIJHT9+XPPnz/dpGZvNposuukjXXHON1q9fr4EDByo1lTZzAIDoQ5ADsYTaSAgEfVoAsDpTIdyPPvpIBQUFOv/889WzZ0+flzvrrLM0cOBA5eXlKTs723QmAQAAUJXrZTU3t1B5eUUEO2Ca02movNwpw+DcAWA9poIdn3/+uWw2my6//HLTKxo5cqQMw9Cnn35qelkAAADU5HpZJdAB1I0mX0B8MRXs2Lp1qySpb9++plfUq1cvSdKWLVtMLwsAAGIXLxgAIokmX0B8MtVnR25urho0aKAGDcwXWlzLHT161PSyAAAg9vCCASAaBNoBNQFbIDaZCnY4nU7Z7YH11Ox0OgNaHgAAxAZGuAEQLfy5BxGwBWKbqchFenq6ioqKdPz4cdMrOn78uAoLC5Wenm56WQAAEJvoUwJArKITYCC2mQp2dOrUSZK0atUq0ytyLfOHP/zB9LIAAAAAEG4EbIHYZSrY0bdvXxmGoVmzZple0axZs2Sz2dSvXz/TywIAAAAAEM3o3yW6mAp2jBw5UvXr19fGjRs1ZcoUn5ebMmWKNmzYoJSUFL+GrQUAAAAAIBrRv0t0MhXsyMzM1E033STDMDR37lyNGzdO3333ndf5v/32W40dO1Zz586VzWbTjTfeqCZNmgScaQAAAAAAooFh0L9LNDI1Gosk/f3vf9eOHTu0dOlSff311/r666+VmZmpLl26uDsfzcvL07Zt29zDzBqGoSFDhmjChAlBzTwAAABQF6qWAwg1Rh2LPqaDHZL0r3/9SyeffLJmzJih4uJiHTlyRF9//XWVeQyj4kDXr19fN998s8aPHx94bgEAAAAfUbUcAOKXX8EOSRo/fryuuOIKLV68WN9884127NihvLw8SRVD1J500knq16+fRowYoYyMjGDlFwAAAPCJa+hQu93GV1cAiDN+BzskKSMjQ+PGjdO4ceOClR8AAAAgaAhyAEB8MtVBKQAAAAAAQLQj2AEAAAAAYUKHuUB4mGrGsn///qCsNCsrKyjpAAAAAEAsoMNcILxMBTsGDBggm80W0AptNpu2bNkSUBoAAAAAEEvoMBcIL9MdlLqGlAUAAAAQ+1w1Dlz/R+gQ5ADCx3Sww2azqXXr1ho+fLh69eoVijwBAAAACBNXjQPX/wHACkwFO/r376+vvvpKv/zyi6ZNm6a2bdtqxIgRGj58uFq0aBGqPAIAAAAIIYIciAd0DhtfTI3G8vLLL+vTTz/V3Xffrfbt22vPnj2aOnWqBgwYoBtuuEEfffSRSktLQ5VXAAAAAABMKTlRLqfToHPYOGO6GUvz5s01fvx4jR8/XuvXr9eiRYuUnZ2t//73v/rqq6/UqFEjXXrppRo+fLi6desWijwDAALElw0AABAvCotLZbfblJ9frLIyB8GOOGGqZkd1Z511lh5//HF99dVXmjx5ss4880wdO3ZMr732mkaOHKnLLrtMr776qnJzc4OVXwBAABj2DgAAxCvKPvEloGCHS/369XX55Zfr9ddf1/Lly3XTTTepRYsW2r59u5544gnNnDkzGKsBAATI1Qldbm6h8vKKeOADAADAkoIS7KisXbt2uvzyy3XJJZcoKSkp2MkDAALkdBoqL+fLBgAAAKzLdJ8d3hQXF+ujjz7SokWL9O2330qSDMPQySefrH79+gVrNQAAAAAAALUKONixbt06LVq0SMuWLVNxcbEMw1Djxo01ZMgQjRgxQqeddlow8gkAAAAAAOATv4Idhw4d0pIlS7RkyRLt2bNHhmHIbrfrT3/6ky6//HINGjRIycnJwc4rAAAAACAIGJkNVmcq2PHhhx9q8eLF+uabb+R0OmUYhtq1a6fhw4dr+PDhatmyZajyCQAAAAAIECOzIV6YCnbcfffdstlsSklJ0eDBg3X55ZerZ8+eocpbSG3ZskUvv/yy1q5dq/z8fDVv3lznn3++JkyYoMzMTNPp/frrr/rqq6+0efNmff/999q6datOnDih3r17a/78+bUuO2DAAO3bt6/WeTZt2qR69eqZzhcAAAAAuLhGZrPbbXI6DYIdsCy/mrHUr19fa9as0Zo1a0wva7PZtGLFCn9WGzTLly/X3XffrbKyMjVp0kSdO3fWrl27NH/+fGVnZ+vNN99U27ZtTaX5wQcf6IknnggoXyeffLLS0tI8TrPZbAGlDQAAAACSCHIgLpgOdhiGoaNHj/q9wki/tB86dEj33XefysrKNGHCBN1yyy1KTExUQUGB7rrrLn355Ze688479fbbb5vKa1pamv70pz+pe/fu6t69u7Zs2aLp06ebyttDDz2kPn36mN0kAAAAAABQialgx6233hqqfITNrFmzVFxcrF69eumOO+5w/96wYUM988wzGjhwoDZv3qxPP/1UAwYM8DndkSNHauTIke6/Dx06FNR8AwAAAAAA38RdsGPZsmWSpFGjRtWY1rhxYw0ePFhvvfWWPvroI1PBDgAAAAAAEB386rMjVh04cMBd46JXr14e5+nZs6feeustbdy4MZxZkyT95z//0ezZs1VSUqKmTZuqZ8+euvTSS7324wEAAKITQzoCABBZcRXsyMnJkSQlJSV5HSbX1THp3r17VVZWpqSkpHBlTx9++GGVv5cuXaqpU6fqmWee0dlnnx22fAAAAP8wpCMAANEhroIdeXl5kiqaq3jrfDQ9PV2S5HQ6dfz4cWVkZIQ8X71791bfvn3VvXt3ZWVlqaysTOvXr9cLL7ygLVu26Oabb9abb76pbt26BbyuxER7EHIMAFUlJNhr/RuIJwUFJe4hHe12m+z26BtRjWvWu8r7Itz7hePiHfsG/gjVeRPJ87GudUdz3sItroIdJ06ckKRaa2skJyfXmD/UnnzyySp/169fX+eff7769eunq666Sj/88IOefvppzZ07N6D12O02ZWRQrRZA6DVqVD/SWQBgAtesZ5HeL5FefzRj38AfoTpvInk+1rXuaM5bqMVMsGPy5Ml69dVXTS/Xu3dvzZ8/X5JUr149SVJZWZnX+UtLS93/d80fKSkpKbrzzjt14403avXq1Tp27JgaN27sd3pOp6H8/KIg5hAAKiQk2Ks80PLzi+VwOCOYIwC14Zr1rvK+Cfd+4bh4x76BP0J13kTyfKxr3dGWtwYN6kWshkfMBDtSU1PdTUzMqNy5pytQcOzYMRmG4bEpi6upi91uj4qOQc8880xJFc1q9u7dG1CwQ5LKy3koAAg9h8PJ/QaIIVyznkV6v0R6/dGMfQN/hOq8ieT5WNe6I523SIqZYMddd92lu+66K6A0OnToIKmiZseBAweUlZVVY569e/dKktq0aRPWzkm9qZwHh8MRwZwAAAAgHBjNBwACF1c9+2RlZal58+aSpHXr1nmcx/X7GWecEa5s1Wr79u3u/3sbQQYAAACxj9F8ACB44irYIUkXXnihJGnhwoU1ph07dkzZ2dmSpMGDB4c1X97MnDlTknTSSSepRYsWEc4NAAAAQsXpNJSXV6Tc3ELl5RUR7ACAAMRdsGPcuHFKSUnR2rVrNXXqVHfTkIKCAk2cOFEFBQXq2rWrBgwYUGPZK6+8UgMGDAh4VJTKXnnlFc2fP1+5ublVfs/NzdXDDz+sZcuWSZJuv/32oK0TAAAA0cnpNFReTq0OAAhUzPTZESytWrXSlClTNHHiRE2fPl0LFixQy5YttWvXLhUVFalp06Z6/vnnPXZeeujQIe3bt08FBQU1ph04cEDDhg1z/+0a1eXbb79Vnz593L/fcMMNuvHGG91/Hzx4UK+++qomT56s1q1bKzMzUyUlJdq5c6fKy8tlt9t19913u2ukAAAAAACA2sVdsEOqaKLStm1bvfTSS1q3bp22b9+u5s2ba8SIEZowYYKaNGliOk2Hw+EeyaWy8vLyKr+XlJRUmT5kyBBJ0qZNm7R//35t27ZNCQkJatOmjXr37q2rrrpKp556qun8AAAAAAAQr+Iy2CFJ3bp10wsvvGBqmZUrV3qd1qZNG/3444+m83HGGWdETWeoAAAAAABYQdz12QEAAAAAAKyNYAcAAECcapbeINJZAAAgJAh2AAAAxBmn05DD4dTwQd3lcDDyBwBrs2Jg14rbFGxx22cHAABAvHI6DeXlFclut8npNAh2ALAkKwZ2rbhNoUKwAwAAIA4R5ABgdVYM7Fpxm0KFYAcAAAAAwJKsGBCw4jaFAn12AAAAAAAASyHYAQAAAAAALIVgBwAAAAAAsBSCHQAAAAAAwFIIdgAAAAAAAEsh2AEAAAAAACyFYAcAAAAAALAUgh0AAAAAAMBSCHYAAAAAAABLIdgBAAAAICY0S28Q6SwAiBEEOwAAQUMhFAAQCk6nIYfDqeGDusvhcMrpNCKdJQBRLjHSGQAAxD4KoQCAUHI6DeXlFclut8npNHjOAKgTwQ4AQMAohAIAQo3nCwAzCHYAAIKCQigAAACiBX12AAAAAAAASyHYAQAAAAAALIVgBwAAAAAAsBSCHQAAAAAAwFIIdgAAAAAAgKBolt4g0lmQRLADAAAAAAAEyOk05HA4NXxQdzkczoiP0sfQswAAAAAAICBOp6G8vCLZ7TY5nQbBDgAAAAAAEPuiIcjhQjMWAAAAAABgKQQ7AAAAAACApRDsAAAAAAAAlkKwAwAAAAAAWArBDgAAAAAAYCkEOwAAAAAAgKUQ7AAAAAAAAJZCsAMAAAAAAFgKwQ4AAAAAACykWXqDSGch4gh2AAAAAABgAU6nIYfDqeGDusvhcMrpNCKdpYhJjHQGAAAAAABA4JxOQ3l5RbLbbXI6DYIdAAAAAAAg9sV7kMOFZiwAAAAAAMBSCHYAAAAAAABLIdgBAAAAAAAshWAHAAAAAACwFIIdAAAAAADAUgh2AAAAAAAASyHYAQAAAAAALIVgBwAAAAAAsBSCHQAAAAAAwFIIdgAAAAAAAEtJjHQGImXLli16+eWXtXbtWuXn56t58+Y6//zzNWHCBGVmZppKyzAMfffdd1q5cqXWr1+vnTt36vjx42rYsKG6du2qYcOG6dJLL5XNZvOaRmFhoV5++WUtW7ZM+/fvV2pqqk4//XSNHTtWffr0CXRzAQAAAACIGzbDMIxIZyLcli9frrvvvltlZWVq0qSJWrZsqV27dqmoqEjNmjXTm2++qbZt2/qc3jfffKMxY8a4/27btq0aNWqkffv2KS8vT5J03nnn6cUXX1RycnKN5Y8ePaqrrrpKu3btUnJysk466SQdPXpUBw8elM1m06RJk3T11VcHutlyOJw6erQw4HQAAACsKjHRroyMBpKk3NxClZc7I5wjANHGdZ94+e1VGj+yL/eKWmRmNlBCQmQalMRdM5ZDhw7pvvvuU1lZmSZMmKAvvvhCixcv1hdffKFzzz1Xv/76q+68806ZiQEZhqE2bdrowQcf1Ndff60VK1Zo8eLFWr16taZMmaLk5GR99tlnmjp1qsflH3zwQe3atUvdunXTihUrtGTJEn322Wd65JFHZBiGJk+erK1btwZrFwAAAAAAAtQsvUGks4BaxF2wY9asWSouLlavXr10xx13KDGxoiVPw4YN9cwzz6hhw4bavHmzPv30U5/T7NGjh7Kzs3XdddepSZMmVaYNGzZMt9xyiyTp7bffltNZNeK3ZcsWrVy5Una7Xc8995xatGghSbLZbBo9erSGDh0qh8Oh6dOnB7LZAAAAAIAgcDoNORxODR/UXQ6HU05n3DWWiAlxF+xYtmyZJGnUqFE1pjVu3FiDBw+WJH300Uc+p5mWlqakpCSv0/v37y9JysvL09GjRz3mp2/fvmrfvn2NZUePHi1J+vzzz1VUVORzngAAAAAAwed0GsrLK1JubqHy8ooIdkSpuAp2HDhwQIcOHZIk9erVy+M8PXv2lCRt3LgxaOstKSlx/z8lJaXKtA0bNlRZb3U9evRQcnKyTpw4QVMWAAAAAIgCTqeh8nJqdUSzuAp25OTkSJKSkpLUsmVLj/O4Oibdu3evysrKgrLeDz74QJLUpUsXpaWlecxTu3btPC6blJSkVq1aSZJ27doVlPwAAAAAAGBlcTX0rGtklMaNG3sdBjY9PV2S5HQ6dfz4cWVkZAS0zs2bN+s///mPJGn8+PE1ph87dsydJ29c0/Lz8wPKi1TRczAAAAA8qzxqQKRGEAAABC6ugh0nTpyQpFr716g8NKxrfn/99ttvuu2221ReXq6//OUvGjJkSEB5qtwcxh92u809lBoAAABq16hR/UhnAQDgp5gJdkyePFmvvvqq6eV69+6t+fPnS5Lq1asnSbU2TyktLXX/3zW/PwoKCnTjjTdq//796tatm5588kmP89WrV0/FxcU+5al6fx9mOZ2G8vPp5BQAAMCbhAS7O8iRn18sh8NZxxIAAG8aNaofsVpyMRPsSE1NdTcxMaNyHxmu5iDHjh2TYRgem7K4mrrY7fYa/Wv4qrCwUDfccIO2bNmizp0765VXXvGaVqNGjVRcXOxuzuKJa1qjRo38yk9l5eU8sAEAAHzhcDgpOwFAjIqZYMddd92lu+66K6A0OnToIKmiZseBAweUlZVVY569e/dKktq0aVNr0xJviouLddNNN2nDhg3q0KGD5syZU2u/Hx06dNChQ4e0e/duj9PLysq0f//+KvkHAAAAAADexVWvS1lZWWrevLkkad26dR7ncf1+xhlnmE7/xIkTuvnmm7V27Vq1bt1ac+fOVbNmzWpdxrWe9evXe5y+adMmlZWVqV69ejr11FNN5wkAAAAAgHgTV8EOSbrwwgslSQsXLqwx7dixY8rOzpYkDR482FS6ZWVluu222/TNN9+oRYsWmjdvnnvIWF/ys3r1ao+1OxYsWCBJ6t+/vxo0oHNRAAAAAADqEnfBjnHjxiklJUVr167V1KlT5XA4JFV0KDpx4kQVFBSoa9euGjBgQI1lr7zySg0YMEBz586t8rvD4dDEiRP1+eefq1mzZpo3b57atm3rU366deum888/Xw6HQ3fddZcOHz4sSTIMQwsWLNC7774ru92um2++ObANBwAAQJ2cTkMOh1MOh1NOpxHp7AAA/GQzDCPu7uLZ2dmaOHGiysvL1aRJE7Vs2VK7du1SUVGRmjZtqjfeeEPt27evsdyAAQO0b98+3Xrrrbrtttvcvy9dulQTJ06UJLVu3VotWrTwuu5Jkyapa9euVX47evSorrzySuXk5Cg5OVknnXSScnNzdeDAAdlsNj344IO69tprA95uh8Opo0cLA04HAADAyuz2ik7sCXYAQGAyMxswGks4DR48WG3bttVLL72kdevWafv27WrevLlGjBihCRMmqEmTJqbSqzxc7b59+7Rv3z6v8xYUFNT4LTMzU4sWLdLMmTOVnZ2tHTt2KDU1Vf3799e4cePUt29fU/kBAACA/whyAEDsi8uaHfGKmh0AAAAAgHCJZM2OuOuzAwAAAAAAWBvBDgAAAAAAYCkEOwAAAAAAgKUQ7AAAAAAAAJZCsAMAAAAAAFgKwQ4AAAAAAGApBDsAAAAAAIClEOwAAAAAAACWQrADAAAAAABYCsEOAAAAAABgKQQ7AAAAAACApRDsAAAAAAAAlkKwAwAAAAAAWArBDgAAAAAAYCkEOwAAAAAAgKUQ7AAAAAAAAJZiMwzDiHQmEB6GYcjp5HADAAAAAELPbrfJZrNFZN0EOwAAAAAAgKXQjAUAAAAAAFgKwQ4AAAAAAGApBDsAAAAAAIClEOwAAAAAAACWQrADAAAAAABYCsEOAAAAAABgKQQ7AAAAAACApRDsAAAAAAAAlkKwAwAAAAAAWArBDgAAAAAAYCkEOwAAAAAAgKUQ7AAAAAAAAJZCsAMAAAAAAFgKwQ4AAAAAAGApiZHOAEJv1apV+r//+z9t2LBBJ06ckM1mk8PhcE9PSEiQ3W6X0+mUzWZTeXl50NadnJyssrIyGYYRtDQBAAAAANZTr1492Ww2GYahrKwsDR48WOPHj1dqaqrptGwGb6GWNn/+fE2ePNnnYENCQkKVQAgAAAAAAOGUlJQkm82m0tJSderUSW+88YbS09NNpUEzFgvbvHmzHn/8cUnSsGHD1LVrVzVs2FBDhgxRkyZNJElpaWnavHmzpkyZouTkZI+BjoSEBJ1++unuk8tms9W5brv991PLNb+n5VzzJScnm9s4E+rKb6DrzsjIUIcOHWqdp1mzZn6n36pVq1qnZ2VlaejQoTV+P+2007wuk5GRUeXvnj176pJLLql1PQ0bNqzxW1JSUq3LuHTs2FGS1LhxY5/m94cv52WgfN1eIJQq31/NSkwMfYXO+vXrh3wdAIDgCmVZHL4JR1m2Xr16QU2vcpnE1+d/u3btJP1+zjVo0MDjfGVlZTr11FPVrVs3/fzzz5o0aZL5/JleAjFj+vTpcjqdGjp0qKZMmaL58+frm2++0bPPPqvRo0dLko4fP66ff/5Zw4YN0y233OJetlGjRu7/Z2RkaPbs2Tp27JgkuWuJtGjRwj1Pt27dqqzb6XS6/3/11VdLklq3bi1JVaoguS6QHj16eNyG3r17u//v7eKsfHP29BJQ10U9duzYGr+1bdu21mUqy83N1R133FHrPL/++qvX/NXl4MGDtU5v2bKldu/eLen3m2Tz5s3VuXPnGvO6pldvqrRjxw6NGDGiym/Vb14FBQWSfn/ht9vtKisrc8/j7UYlSbt27VLr1q21fPnyGi9bdf3tiafgkuu8DPZNvLLK2xtpngJHdT0kaws8un6rbf/XVRCqLRgUjEBRYmKiUlJSvE5PSEgIeB0u/u6jSKl8vbr+X9u+8hS8rKy2baw8rfp+atiwoe65555a065LNO5fIF657idcl9Z3ww03RDoLIRXIh4JwCUeDiyeffDKo6VV+5ysuLpakOpub7N27V9Lv7yNdu3Z1Txs2bJj69esnqeK+s3HjRl199dWy2+1avny5tm3bZip/0X/U4ZfCwkJ9+eWXkqRRo0ZJqqjF4XrhcNXskKTs7GxJ0p/+9Cf3b9ULs6WlpTUuwMrznHHGGR7zUa9ePfdXfU8vIq6TvLCw0OPyrmWlipd6T0pLS93/r3zBuZSUlHhczsXTS3ptLwme1PXi4LrBespfXXwpYLhq5LiOUevWrbV161av+Th+/HiV3/Py8vTSSy9V+a1yXitHal3rqv6yXddDctiwYUpPT6+xPdUfPr70GXPBBRd4neYKqkVCMCLyvp57rgdKZXU9JF3TPc3n+i2YffZUFozmcUlJSWratKnX6bUFc8zylIZrH4Xjy0ttPB2/yter65rylE/X8a1tP1aer65p1fPizz2uOmqGANHDdU17uyekpaWFMzsIIbPNA2JNMJ5PVhBIbXNf1fWBy1V2cDqdstvt+v77793TRo0a5a754fqwvm7dOvXt21fS7++tviLYYVFbt25VaWmpkpOTvdaacNm4caMkacGCBe7fjh496v5/Xl6exo8fX+PEdb0wJycne33BrCsI4LJr1y6Pv69YscL9/z179viUlllLliypMz+uQI23qPDatWtrXYevVQM9vYBUrkHjybffflvlJiFVNGHyFPl03eg9vSytXr3a6zoqnw+uNHJzc6vMM2fOnFrz+dprr2nkyJE1gk+uAlTlFzNPL2muIEBCQkKtNSxctVzMCHe0v7aXZV/PlUjUMgkkEBKMQkZycrLy8/O9Tg9mf0O1nRPRGOyozHWcagv0+nOd+KKwsFAzZswIKI26AtTBFOljCcS66h9PELtcNXhhbQ899JCp+WsrD3mb5moN4E3lD+ApKSnu535ycrK6deumDRs2SJLatGkjqeJd56yzzpL0+3urrwh2WJTrZT0rK6vO6NquXbv0888/a/HixR6nl5WV6fvvv6/xcuV66WjUqJEWLlwoqWa1JV+/Unsr3B45csT9f28FfNdoMpLn4EpdVal27txZ47fqL3Wul6iTTz7ZYxozZ86sdR2+Ft4HDRpU47f9+/f7tGxl3l6EQ1k9rraXUKnixlc9KCN5DsB4yueJEyckSRdeeKE++OADr+vx54U3WNH+YOxfb7WcQrEus+raT54CV8FUXl5e63lWW80Vf9bl7SFeuTaZv8Lxkl3bfgjlF67ffvstoOXDGcijj/b4FKraX6hbbR8zEFmvvPJKpLOAMMjJyTE1f23lhcotBVzqqjkqVS2rFxUVVVn2H//4h3788Uf17t3b3TR979697sCHtw/k3hDssChXRM2XDiF//fVXXXzxxV5P5sTERP3lL3+p8burkPjbb7+5C7fV1+dvG3ozhYjWrVu75/f0haHyRRQob+3EarsR2O12n/ZDenq6vvnmG1NpS1X7V4lldRV2XOfbgAEDdPjw4XBkKWRqe8GK5dGQXNsVqrbdvgaCgiWUAQFesmtH/wAIpWBcf1zDvqt8PXvab+GszQXvgllehvWdeeaZVT5Ku9T1wcP1zuZpIIX9+/frv//9ryZOnKgpU6Zo/fr1kirKxq6az3XVGqmOYIdFub6C+9IpYOVaDJ6q0Ddr1kzTpk3z2umc3W53BxlcHXFWnuaPyg/Dc845p9aC7549e+RwOJSUlBSVhQ9vo9xUV1xc7FfV8gceeKBK57KxypeXm2bNmun1118PQ27gia8voKHq98ObeHsxrisYHAv7o64mjuE+h+KRt36wEF5WqCFS1wedUNf6A/sW4ffjjz/69VHI9QF98+bNHvvxO378uBYt+v/au/PYJss4DuDfdu3oZndSdoEEVBhXxmDAMJsRBsEhcoWoAYVMiCRckmkCDBH1jwkGlcgGGI8EAwEUjGgM9wSVcyPbmAyYCIoytjHcOnd17dr6x/K+9nh77Wi77vtJlrXv1adv3/d5n/f3Psc3WLlypeRDLuEe110MdgQoodqPJ9WBR4wYgYsXL2LdunVW04UMtLKyUnI9pVIpPpW3LaB6chJMmTLFblr//v1RXV3tVsHXYDBApVI5HY1D6ibA1bCxtut7eiOh0+lcNqUBXJ+8wjakLmgzZ850Ky2W6zrrxLCnSX2mO+1+a2trUVJS4nAbgu7og6MvFxxGjBghOd3d/dqVoGNnRtPx5MY4EDq/dLfPDl9zlu+xozjv0Wg0ktctDqXtXY7KDv74kMZT7tZIDITv6mtRUVGS5y73rWv+Vq7zt/R4yp3atlFRUXbTTp48KZYPbMsrcrkcGo0Gf/75J27cuIGMjAyreYDn5UQGOwKU0JzE3ao+8fHx2LNnD9RqtdXwP0BHjYObN2/i4MGDkiemXq93eMK6ewGMj4/HU089ZTddLpfj999/d2sbwnYcBQ1UKhXa29vtChxZWVl20xwNo9re3u607xBHuuPmQ6h1I3XDuWPHDqfrCt/PMu3C9rx5gRSiuV39zKCgIKe/Q3fcSEltPyUlBRMmTJBcvrdftCw56qnbG8eKpxF7T0mNYtMZ/tzG3F+ORWfnYXc0SfJGx8Lx8fE9/hk9rV+/fpLXIGHov97K09/f1zWenPUDROSu+vp6yeuPu52bO6NQKLx2/fDF+ehp03qp81WlUnXbPgrkAJXw+z777LOS8x01mWpvb4dOpxMHZxAetIeEhIj9pbnTRYMl5roBSqitcP/+fcnaHbbT2tra8NxzzyEtLQ3Z2dlW87RaLRYtWgSz2SyZwSYnJzts4uJuO8za2lp8+OGHktOdETIu4aRy1mmNTqeDXC63y6Q++OADu4Kg5YkkvBbWcxTAcRbY6Y4ODbVareTnbNy4ESdPnnS6rlR1ZV/cEDkb8cWVefPmicefs6huZ/u8kOpkydbVq1dx/fp1yXmu9qcQxXZ1sfWHG1VHv5OrmmLeeFLsD/sH8O825lKBY1/o6X3kjdohUu2RpbiTf/iKo1qZvZ2nv78/1Hhijabe64UXXvB1EkRSo6a4Kme6e6PvrRtwX5wLnuYBUmnU6XS9PkjhjXKUsK+dDSggGD9+vPhaqVTi0qVL2LBhA4D/O1MdPXq0OCqnJzXyAQY7AtbIkSOhVCqh1+tRVlZmNa+trc1qmFmgY2hRoaNR2yFFzWaz+BROKjN1Fp2VypClMo/29na3O0ayPEmNRiMSEhLENLh6amIymexu2KSaT1g+/fXnmxrAvQtTTU2N1fuIiIhue8LtCVcjtjiTnp4u/ha+6kTL2XEqdVxb3vwL+9vVMeoPF9HuCM71FH/YP/7OH27qPCFV8PKXJhbungu+yE+Jegt/CVJ3VlxcnMuHb/7OnWtnV64dnjYtYODPd7xZjhIe1DqTlpYmvjYYDFb3rcI1eNq0aWJnpcnJyR6lgcGOAKVWq5Geng4A4rCwQMdBtGbNGqsaEN999x0qKiqs/hYvXizOj4iIQH5+PgDpp+ZSFzEhgtzW1mZX28IyeCAUaJ1VSbK9ObQ9ScPDw8Vt2maeQtpiY2Nx+vRpVFRU4JVXXrFaZuPGjXafaRnwEarVO8scNm3aJHa4J5XhOxsxxdm84OBgbNmyBUDHaC2A/b7asmULEhISHG5DLpfbBXikqpCnpqY63EZ3VDfs168fnn76aQCdawJw4sQJp9u25Wl14bq6OpfLzJgxA08++SSA/38PQUxMjN3ylvvdbDYjMjLSrX50OttEwt3vLHXOCtNCQ0Md1gzoynHQXTev0dHRLpcZPHiw1f/uJpPJ3O4np6s8LUDKZDKUl5e7XC4oKKjHqhELx6FUW10pUnmgJwXhnmwaMGvWLLeW62wAtqfS7s4DgM70j+MLQvV8jUbTYzfNvbV5SW8JIvTWILWwf5VKJX755Refp8PZceoqP7fNU6W21ZXzwN0mqEL5xhdNQbujHDJ79myn511vOSe94b333kNFRQU2bdrkcBmFQiEGEoUa0F9//TXOnTsHoCPvCA8PR0pKCi5dugQAyMzM9CgdMnNvzYHIpbKyMrHa3bvvvosFCxbg9ddfx4kTJyCTyWA2mxEWFoYrV64AAG7duoW9e/di0qRJ+Oyzz8RhVkNCQpCTk4PNmzdLfs6ECRMgl8tRWFgoTouNjRVrE0RGRkKr1SIhIQH379+3Wlej0YhDFCmVSrsbQSGdUqSWlxIREYFBgwZh5MiRuHfvHgoLC60y/aCgoC4N96lWqzF48GBcv34dcrlcMuCi0WjsngqoVCqxaY2jgn1ERAQ2bNiAnJwccTnb9I4bN07stNPWoEGDUFlZ6bKgMWjQINTV1fl02DGp39rZ728pNDTUKu0KhaJHnm6PHj3arRvJznD3eO5pKpUKer2+U09dnP1ewvFuydmx3xnBwcHQ6/XivhTeu+JpHqBQKGA2m7t9mOCunAP+yN20+/N3tM1bAomz/e4vv4lwbgp9bnm71lJ35FHCd/CXfUr2+Nt4h7Cfe/P+DuRrQnewLE8tWrQIL730EnJzc3HhwgXJ5ZVKJYxGI0wmE1avXo2dO3faHRtvvvkmjhw5gvLyckyfPh07d+70KE0MdgS4PXv2YOvWrZ3KVHpzZkRERERERET+Lzg4GAaDwereMzY2FvX19dDr9Rg6dCj279/vVg1fSwx29AEXL17Erl27rGpeuKJWq6HX6/267X5vYvt0iIEkIiIiIiLqSyxr3cbExKC5uRktLS3ifZFSqURoaChMJhPa2tqQkJCAzMxMLF++3OFomc4w2EFEREREREREAaV39shEREREREREROQAgx1EREREREREFFAY7CAiIiIiIiKigMJgBxEREREREREFFAY7iIiIiIiIiCigMNhBRERERERERAGFwQ4iIiIiIiIiCigMdhARERERERFRQGGwg4iIiIiIiIgCCoMdRERERF20ePFiJCYmIi8vz9dJISIiIjDYQURERF6Ql5eHxMREyb+xY8dixowZWL9+PYqLi3vk8+/du4e8vLyAC0acPn0aiYmJeO2116ymZ2VlITExEaWlpb5JGBERkY8x2EFERERepdFoxL/o6GgYDAbcvXsXR44cwcKFC3skIFFZWYn8/Hzk5+d3+7Z9qbCwEAAwadIkcZper0dJSQlCQ0MxZswYXyWNiIjIpxS+TgARERH1LefPn7d6bzQaUVpaitzcXJSXlyM/Px9paWkYP368j1LYexQVFQGwDnZcvXoVOp0O6enpUChY1CMior6JNTuIiIjIp4KCgpCSkoJdu3aJ0woKCnyYot6hsbERN2/eRHR0NIYNGyZOv3z5MgBg4sSJvkoaERGRzzHcT0RERH4hLi4OkZGR0Gq1aGlpsZtvMBjw888/4+zZsygvL8eDBw+g1WoRFhaGUaNGYf78+Zg1axZkMpnVehkZGaisrBTfJyYmWs2fP38+tm7dajWtpaUFX331FQoKCnDr1i00NzcjOjoagwcPRkZGBubMmQONRiP5PcxmMw4dOoRDhw7h9u3bMJvNGD58OBYtWoS5c+d2dvfYKSoqgslkwsSJE62+M4MdREREDHYQERGRn6ipqYFWqwUADB061G5+cXExVq5cKb5Xq9UIDg5GXV0dzp07h3PnzuHUqVPYvn075PL/K69GRUWhqakJDQ0NAGAXpFCr1Vbvy8vLsWrVKlRVVQEA5HI5wsPDUV9fj5qaGhQVFUEulyMrK8sujUajEatWrUJBQQEUCgVUKhWam5tRWlqK0tJS3L17164zUXctXLgQNTU14vumpiYAHc2CMjIyxOnV1dUAgDfeeEPcD7GxsThw4ECnPpeIiKg3YrCDiIiIfMpoNKKsrAy5ubkAgP79+2PevHl2y4WEhODFF19EZmYmkpKSxCCFVqvF999/j48//hjHjx9HSkoKlixZIq73zTff4PLly+I02z5DLFVVVWHZsmWor69HfHw81q1bh6lTpyIkJARmsxm3b9/G8ePHER0dLbn+/v37YTKZsHXrVsycORMqlQrV1dV45513cObMGezevRtz5szBkCFDPN5PNTU1VjVUBE1NTWLgw/a7EBER9VUMdhAREZFXpaWlia9NJhMaGhpgNBqhVqsxe/ZsZGdnIzw83G69pKQkJCUl2U2PjIzEkiVLEBMTg7Vr12Lv3r1WwQ5PfPTRR6ivr0dkZCQOHDiA+Ph4cZ5MJsMTTzyB1atXO1y/oaEBX375JSZPnixOi4uLw44dOzBt2jQ8ePAAx44dw4oVKzxO248//ii+bmxsRGpqKsLCwnDx4kWxBkd+fj7y8vKwdu1aq1owREREfQ07KCUiIiKvevjwofhXV1cHo9EIANDpdGhqasI///zTqe1OmTIFAPDXX3+htrbW4/VbWlpw7NgxAMDy5cutAh3uGj9+vFWgQxAcHIz09HQAQEVFhcfbtVVUVASj0YgJEyZYNdmRGoqWiIioL2LNDiIiIvIq25v9trY23LlzB/v27cPhw4dx/vx5bN++HdOnT7dbt6mpCQcPHsTZs2dx+/ZtNDY2wmAw2C1XXV2NAQMGeJSua9euiduaOnWqR+sKxo4d63BeTEwMAIh9h3SFENRITU0Vp+n1ely9ehUqlUqyBgwREVFfwmAHERER+VS/fv0wcuRI5ObmoqGhAadOncKGDRtw9uxZq85D//jjD2RlZYkdcAId/XiEhYWJtRsePnwIAGhtbfU4HcK6ADBw4MBOfZdHHnnE4TyFoqPY1d7e3qltWxJGXLGswfHrr79Cp9Nh8uTJCA4O7vJnEBER9WYMdhAREZHfeP7553Hq1Ck0Njbip59+wqxZs8R5OTk5qK6uxsCBA7Fu3TpMnjwZkZGR4nyj0YhRo0YB6Bj+NVAUFxdjzZo1VtOEpj5Lly4Vh51ta2sDAJSWllr1i7J06VIsW7bMS6klIiLyDwx2EBERkd+wrFFx79498XVVVRVKSkoAdHQimpycbLeuZc2MzrBs9lJZWYnHHnusS9vrLgaDweF3k+rfRKfTQafTie9bWlp6LG1ERET+isEOIiIi8hu2TVQElsOoCrU3bF24cMHhdi078TSbzWJtCEtjxoyBUqmEwWDAmTNn/CbYkZqaatXPybZt2/D555/jrbfewssvvwygo7+OSZMmwWQy4cqVK2zGQkREfR5HYyEiIiK/8cMPP4ivx4wZI74OCwsTX9+8edNuvaamJuzevdvhdi37/vj3338llwkJCRGbzXz66adWARZ/cunSJQDW/XWUlZWhtbUV48aNY6CDiIgIDHYQERGRH6itrcX27dvx7bffAgCSk5Mxbtw4cf7jjz+OhIQEAMDGjRtx7do1cV5JSQmWLFnidJSTIUOGQKlUAgAOHTrksE+P7OxsREVFQavVYuHChTh69KjYJMRsNuO3337D+++/jyNHjnTp+3ZWY2Mjbty4gejoaAwbNkycLozOIjXsLRERUV/EZixERETkVZadZwIdHWs2NjaK74cPH44dO3ZYNTWRy+XYvHkzVq9ejVu3bmHBggViM5fW1laEhoZi165dyMrKkvzMkJAQzJ07F4cPH8a2bduQn5+PqKgoyGQyPPPMM1i/fj0AIC4uDl988QVWrFiBqqoqZGdnIygoCGFhYWhtbRU7Ac3JyenOXeK2wsJCGI1GTJw40Wr/SA1FS0RE1Jcx2EFEREReZdvZplKpxIABA5CYmIjMzEzMnTtXsinG1KlTsW/fPnzyyScoLi5Ga2srBgwYgJkzZ+LVV1912cfG22+/jfj4eJw4cQJ///037t+/DwCor6+3Wm706NE4evQo9u/fj4KCAty5cwfNzc3QaDR49NFHMW3aNMyePbuLe6FzpIac1ev1KC0tRWhoKJKSknySLiIiIn8jMwfS2GxERERERERE1Oexzw4iIiIiIiIiCigMdhARERERERFRQGGwg4iIiIiIiIgCCoMdRERERERERBRQGOwgIiIiIiIiooDCYAcRERERERERBRQGO4iIiIiIiIgooDDYQUREREREREQBhcEOIiIiIiIiIgooDHYQERERERERUUBhsIOIiIiIiIiIAgqDHUREREREREQUUBjsICIiIiIiIqKAwmAHEREREREREQWU/wC9t1ajaEai/AAAAABJRU5ErkJggg==\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Combine the results across all batches.\n",
        "flat_predictions = np.concatenate(predictions, axis=0)\n",
        "\n",
        "# For each sample, pick the label (0 or 1) with the higher score.\n",
        "flat_predictions = np.argmax(flat_predictions, axis=1).flatten()\n",
        "\n",
        "# Combine the correct labels for each batch into a single list.\n",
        "flat_true_labels = np.concatenate(true_labels, axis=0)\n",
        "\n",
        "# Calculate the MCC\n",
        "mcc = matthews_corrcoef(flat_true_labels, flat_predictions)\n",
        "\n",
        "print('Total MCC: %.3f' % mcc)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "saxee7eYt0lc",
        "outputId": "b32d86a4-8dd8-479b-b1fd-a256e86cc4e3"
      },
      "execution_count": 65,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total MCC: -0.039\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "\n",
        "# Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()\n",
        "\n",
        "output_dir = './BERT_model/'\n",
        "\n",
        "# Create output directory if needed\n",
        "if not os.path.exists(output_dir):\n",
        "    os.makedirs(output_dir)\n",
        "\n",
        "print(\"Saving model to %s\" % output_dir)\n",
        "\n",
        "# Save a trained model, configuration and tokenizer using `save_pretrained()`.\n",
        "# They can then be reloaded using `from_pretrained()`\n",
        "model_to_save = model.module if hasattr(model, 'module') else model  # Take care of distributed/parallel training\n",
        "model_to_save.save_pretrained(output_dir)\n",
        "tokenizer.save_pretrained(output_dir)\n",
        "\n",
        "# Good practice: save your training arguments together with the trained model\n",
        "# torch.save(args, os.path.join(output_dir, 'training_args.bin'))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DAdPyD7qf5Ls",
        "outputId": "2abb908e-1099-4f0f-b0ca-978325f9862b"
      },
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving model to ./BERT_model/\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('./BERT_model/tokenizer_config.json',\n",
              " './BERT_model/special_tokens_map.json',\n",
              " './BERT_model/vocab.txt',\n",
              " './BERT_model/added_tokens.json')"
            ]
          },
          "metadata": {},
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "## check out the file sizes\n",
        "!ls -l --block-size=K ./BERT_model/"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-ymyM0RCo93q",
        "outputId": "4503ea40-8a38-4f74-ef64-95890ad14b77"
      },
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "total 428052K\n",
            "-rw-r--r-- 1 root root      3K Feb 23 15:15 config.json\n",
            "-rw-r--r-- 1 root root 427812K Feb 23 15:15 model.safetensors\n",
            "-rw-r--r-- 1 root root      1K Feb 23 15:15 special_tokens_map.json\n",
            "-rw-r--r-- 1 root root      2K Feb 23 15:15 tokenizer_config.json\n",
            "-rw-r--r-- 1 root root    227K Feb 23 15:15 vocab.txt\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Mount Google Drive to this Notebook instance.\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4E0sM5h6t9cA",
        "outputId": "33efbed1-d9b4-4f3a-82e1-bda3ce67f25a"
      },
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Copy the model files to a directory in your Google Drive.\n",
        "!cp -r ./BERT_model/ \"/content/drive/MyDrive/BERT/BERT_Fine-Tuning_Model/\""
      ],
      "metadata": {
        "id": "dmHPdEVtuLsl"
      },
      "execution_count": 73,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "state = {'model': model.state_dict(), 'optimizer': optimizer.state_dict(), 'epoch': epochs}\n",
        "torch.save(state, '/content/drive/MyDrive/BERT/checkpoint')"
      ],
      "metadata": {
        "id": "7WbxyKF1wTkH"
      },
      "execution_count": 77,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Load a trained model and vocabulary that you have fine-tuned\n",
        "model = model.from_pretrained(output_dir)\n",
        "#model_to_save.save_pretrained(output_dir)\n",
        "tokenizer = tokenizer.from_pretrained(output_dir)\n",
        "\n",
        "# Copy the model to the GPU.\n",
        "model.to(device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z_2NU-7Xzc-_",
        "outputId": "81c4a15e-1d0a-431b-c57f-b44d25000408"
      },
      "execution_count": 83,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BertForSequenceClassification(\n",
              "  (bert): BertModel(\n",
              "    (embeddings): BertEmbeddings(\n",
              "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (token_type_embeddings): Embedding(2, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): BertEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0-11): 12 x BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (pooler): BertPooler(\n",
              "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "      (activation): Tanh()\n",
              "    )\n",
              "  )\n",
              "  (dropout): Dropout(p=0.1, inplace=False)\n",
              "  (classifier): Linear(in_features=768, out_features=41, bias=True)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 83
        }
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}