BERT_Classifier.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "BERT Classifier.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "machine_shape": "hm",
      "authorship_tag": "ABX9TyONlw3UflGwQR40BjjETf2D",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "23d6b54a1dff470498abf07a707edd82": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_fac0c28cfaaf45ba9febc2ce3d588e57",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_694d5b5333ef48f09a0261fb7f1121e2",
              "IPY_MODEL_7171dcab92a146859d92a0eb0d7c9644"
            ]
          }
        },
        "fac0c28cfaaf45ba9febc2ce3d588e57": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "694d5b5333ef48f09a0261fb7f1121e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e9779525495d45c7a3854e181609d4f5",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b618beefeed748a4ae15dc1094aed5c7"
          }
        },
        "7171dcab92a146859d92a0eb0d7c9644": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_70e073e788144a8080b98160d2e7438b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:00&lt;00:00, 817kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_348c40458b4a4cfa8d1edbd7ea1a0430"
          }
        },
        "e9779525495d45c7a3854e181609d4f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b618beefeed748a4ae15dc1094aed5c7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "70e073e788144a8080b98160d2e7438b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "348c40458b4a4cfa8d1edbd7ea1a0430": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d01b0aa7b40048af93a25bdafcf56f5b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0aeb351cfc91401ebbd8d83307951554",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_136b6bf92f9f4da2809c81d222f93c8c",
              "IPY_MODEL_b5ddd4651eb049528265aadfaca4df41"
            ]
          }
        },
        "0aeb351cfc91401ebbd8d83307951554": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "136b6bf92f9f4da2809c81d222f93c8c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1dad64454cb0455f8255e730454bf00b",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 433,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 433,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b61560a3de7c4d43a2f97b3613d334e5"
          }
        },
        "b5ddd4651eb049528265aadfaca4df41": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8a15be73d30349398255c3e51f440d18",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 433/433 [01:06&lt;00:00, 6.49B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5911b7dc0507455ebac902d25daf4981"
          }
        },
        "1dad64454cb0455f8255e730454bf00b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b61560a3de7c4d43a2f97b3613d334e5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8a15be73d30349398255c3e51f440d18": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5911b7dc0507455ebac902d25daf4981": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0b1fcf138b98437f95480d2ae106e8ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4df77559eeec49b29d983b714ac4347c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5b3ad5fc37184afab8b752b58869e16c",
              "IPY_MODEL_b0f84fc90b0a48c7beebb7843cbb7469"
            ]
          }
        },
        "4df77559eeec49b29d983b714ac4347c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5b3ad5fc37184afab8b752b58869e16c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_51fb22e38c8d43f8a558833c25b40667",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 440473133,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 440473133,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_38f9f520bf744db68fe1bf6549f2d1a7"
          }
        },
        "b0f84fc90b0a48c7beebb7843cbb7469": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a1c0421189374d51adee47e0a98c240e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 440M/440M [00:34&lt;00:00, 12.8MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1849991a582c4885bdd1e9c6dad04861"
          }
        },
        "51fb22e38c8d43f8a558833c25b40667": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "38f9f520bf744db68fe1bf6549f2d1a7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a1c0421189374d51adee47e0a98c240e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1849991a582c4885bdd1e9c6dad04861": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mbmackenzie/peace-speech-project/blob/master/BERT_Classifier.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PCkgHWwzWMoH",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cc9867a0-62e9-4fea-9a5a-9a8b2ca1c65d"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "39_9-AScOYlP"
      },
      "source": [
        "# About the Notebook\n",
        "\n",
        "This notebook implement BERT with Linear layer for the classification purpose.\n",
        "\n",
        "1. [Data](#section_1)\n",
        "2. [Model](#section_2)\n",
        "3. [Random Sampling](#section_3)\n",
        "4. [Random Sampling Test data](#section_4)\n",
        "5. [Separate Prediction model](#section_5)\n",
        "6. [Evaluation Metrics](#section_6)\n",
        "7. [Save BERT Classifier](#section_7)\n",
        "8. [Load BERT Classifier](#section_8)\n",
        "\n",
        "\n",
        "\n",
        "## 1. Data <a class=\"anchor\" id=\"section_1\"></a>\n",
        "\n",
        "We have used the sampled data of countries from peaceful and non-peaceful nation. For the BERT, we have chosed the n-gram preprocessed text data (no lemmatization and stopword removal). Since BERT takes too much time for the fine-tuning and prediction for evaluation, instead of using whole 874,535 data, we have randomly sampled 100,000 data for train & validation and 50,000 for test. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DFAsdUN0WZMK"
      },
      "source": [
        "#huggingface library installation\n",
        "!pip install transformers\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import os\n",
        "from transformers import BertForTokenClassification, AdamW, BertConfig, BertForSequenceClassification, get_linear_schedule_with_warmup\n",
        "import random\n",
        "import transformers\n",
        "from torch.utils.data import TensorDataset, random_split,DataLoader, RandomSampler, SequentialSampler\n",
        "import torch.nn.functional as F\n",
        "from torch.autograd import Variable\n",
        "import time\n",
        "import datetime\n",
        "from platform import python_version\n",
        "import sklearn\n",
        "import torch\n",
        "import random\n",
        "\n",
        "from torch import nn\n",
        "#Using Colab GPU for training\n",
        "import tensorflow as tf\n",
        "\n",
        "# Get the GPU device name.\n",
        "device_name = tf.test.gpu_device_name()\n",
        "\n",
        "# The device name should look like the following:\n",
        "if device_name == '/device:GPU:0':\n",
        "    print('Found GPU at: {}'.format(device_name))\n",
        "else:\n",
        "    raise SystemError('GPU device not found')\n",
        "\n",
        "#To confirm that we are using GPU for the training later\n",
        "\n",
        "# If there's a GPU available...\n",
        "if torch.cuda.is_available():    \n",
        "\n",
        "    # Tell PyTorch to use the GPU.    \n",
        "    device = torch.device(\"cuda\")\n",
        "\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "\n",
        "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
        "\n",
        "# If not...\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")\n",
        "\n",
        "\n",
        "directory = '/content/drive/My Drive/n_gram_processed'\n",
        "peace = ['AU', 'CA', 'IE', 'NZ', 'SG', 'GB']\n",
        "non_peace = ['BD', 'KE', 'NG', 'PK', 'TZ']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pz9PAToNYQRu"
      },
      "source": [
        "df_whole = pd.read_csv('/content/drive/My Drive/whole_df.csv',index_col=[0])\n",
        "\n",
        "#add labels to the dataframe\n",
        "df_whole['label'] = -1\n",
        "for i in range(len(df_whole)):\n",
        "  if df_whole['country'][i] in peace:\n",
        "    df_whole['label'][i] = 0\n",
        "  else:\n",
        "    df_whole['label'][i] = 1\n",
        "\n",
        "df_whole = df_whole.dropna()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OxHC6fiFPyEh"
      },
      "source": [
        "## 2. Model <a class=\"anchor\" id=\"section_2\"></a>\n",
        "\n",
        "We have used BERT-base-uncased for the BERT model. Instead of separate each sentence, we have treated the whole document as a sentence, hence add [CLS] and [SEP] at the beginning and end of the document. Since some of the documents has longer token lengths than 512, we have first tried with truncation (only using 510 tokens at the beginning). Although the best method is to use some part from head and some part from tail according to the [article](https://arxiv.org/abs/1905.05583), we have achieved 94% accuracy for both validation and test, hence will not use the method.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BZGjBe97ehhk",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "23d6b54a1dff470498abf07a707edd82",
            "fac0c28cfaaf45ba9febc2ce3d588e57",
            "694d5b5333ef48f09a0261fb7f1121e2",
            "7171dcab92a146859d92a0eb0d7c9644",
            "e9779525495d45c7a3854e181609d4f5",
            "b618beefeed748a4ae15dc1094aed5c7",
            "70e073e788144a8080b98160d2e7438b",
            "348c40458b4a4cfa8d1edbd7ea1a0430"
          ]
        },
        "outputId": "44820817-f910-424e-af86-d69e63988092"
      },
      "source": [
        "from transformers import *\n",
        "\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "23d6b54a1dff470498abf07a707edd82",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3ibmMaeGe6Dl"
      },
      "source": [
        "MAX_LEN = 512\n",
        "\n",
        "#The below code can be later modified to add the segment_id of setences. \n",
        "#However, since the summary + title data outperforms the validation accuracy from finBERT, I will modify this after implement the XLNet classifier\n",
        "\n",
        "def preprocessing_for_bert(data):\n",
        "    \"\"\"Perform required preprocessing steps for pretrained BERT.\n",
        "    @param    data (np.array): Array of texts to be processed.\n",
        "    @return   input_ids (torch.Tensor): Tensor of token ids to be fed to a model.\n",
        "    @return   attention_masks (torch.Tensor): Tensor of indices specifying which\n",
        "                  tokens should be attended to by the model.\n",
        "    \"\"\"\n",
        "    # Create empty lists to store outputs\n",
        "    input_ids = []\n",
        "    attention_masks = []\n",
        "\n",
        "    # For every sentence...\n",
        "    for sent in data:\n",
        "        # `encode_plus` will:\n",
        "        #    (1) Tokenize the sentence\n",
        "        #    (2) Add the `[CLS]` and `[SEP]` token to the start and end\n",
        "        #    (3) Truncate/Pad sentence to max length\n",
        "        #    (4) Map tokens to their IDs\n",
        "        #    (5) Create attention mask\n",
        "        #    (6) Return a dictionary of outputs\n",
        "        encoded_sent = tokenizer.encode_plus(\n",
        "            sent,  # Preprocess sentence\n",
        "            add_special_tokens=True,        # Add `[CLS]` and `[SEP]`\n",
        "            max_length=MAX_LEN,                  # Max length to truncate/pad\n",
        "            pad_to_max_length=True,         # Pad sentence to max length\n",
        "            #return_tensors='pt',           # Return PyTorch tensor\n",
        "            return_attention_mask=True,      # Return attention mask\n",
        "            truncation = True\n",
        "            )\n",
        "        \n",
        "        # Add the outputs to the lists\n",
        "        input_ids.append(encoded_sent.get('input_ids'))\n",
        "        attention_masks.append(encoded_sent.get('attention_mask'))\n",
        "\n",
        "    # Convert lists to tensors\n",
        "    input_ids = torch.tensor(input_ids)\n",
        "    attention_masks = torch.tensor(attention_masks)\n",
        "\n",
        "    return input_ids, attention_masks"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iW001nUKD8Gf",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ae6cc1f3-0ebd-43ef-8f86-0822703aa883"
      },
      "source": [
        "%%time\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from transformers import BertModel\n",
        "\n",
        "# Create the BertClassfier class\n",
        "class BertClassifier(nn.Module):\n",
        "    \"\"\"Bert Model for Classification Tasks.\n",
        "    \"\"\"\n",
        "    def __init__(self, freeze_bert=False):\n",
        "        \"\"\"\n",
        "        @param    bert: a BertModel object\n",
        "        @param    classifier: a torch.nn.Module classifier\n",
        "        @param    freeze_bert (bool): Set `False` to fine-tune the BERT model\n",
        "        \"\"\"\n",
        "        super(BertClassifier, self).__init__()\n",
        "        # Specify hidden size of BERT, hidden size of our classifier, and number of labels\n",
        "        D_in, H, D_out = 768, 100, 2\n",
        "\n",
        "        # Instantiate BERT model\n",
        "        self.bert = BertModel.from_pretrained('bert-base-uncased')\n",
        "\n",
        "        # Instantiate an one-layer feed-forward classifier\n",
        "        self.classifier = nn.Sequential(\n",
        "            nn.Linear(D_in, H),\n",
        "            nn.ReLU(),\n",
        "            nn.Dropout(0.1),\n",
        "            nn.Linear(H, D_out)\n",
        "        )\n",
        "\n",
        "        # Freeze the BERT model\n",
        "        if freeze_bert:\n",
        "            for param in self.bert.parameters():\n",
        "                param.requires_grad = False\n",
        "        \n",
        "    def forward(self, input_ids, attention_mask):\n",
        "        \"\"\"\n",
        "        Feed input to BERT and the classifier to compute logits.\n",
        "        @param    input_ids (torch.Tensor): an input tensor with shape (batch_size,\n",
        "                      max_length)\n",
        "        @param    attention_mask (torch.Tensor): a tensor that hold attention mask\n",
        "                      information with shape (batch_size, max_length)\n",
        "        @return   logits (torch.Tensor): an output tensor with shape (batch_size,\n",
        "                      num_labels)\n",
        "        \"\"\"\n",
        "        # Feed input to BERT\n",
        "        outputs = self.bert(input_ids=input_ids,\n",
        "                            attention_mask=attention_mask)\n",
        "        \n",
        "        # Extract the last hidden state of the token `[CLS]` for classification task\n",
        "        last_hidden_state_cls = outputs[0][:, 0, :]\n",
        "\n",
        "        # Feed input to classifier to compute logits\n",
        "        logits = self.classifier(last_hidden_state_cls)\n",
        "\n",
        "        return logits"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 54 µs, sys: 0 ns, total: 54 µs\n",
            "Wall time: 57.2 µs\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7WzSE-KQD-cx"
      },
      "source": [
        "def initialize_model(epochs=4):\n",
        "    \"\"\"Initialize the Bert Classifier, the optimizer and the learning rate scheduler.\n",
        "    \"\"\"\n",
        "    os.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"\n",
        "    # Instantiate Bert Classifier\n",
        "    bert_classifier = BertClassifier(freeze_bert=False)\n",
        "\n",
        "    # Tell PyTorch to run the model on GPU\n",
        "    bert_classifier.to(device)\n",
        "\n",
        "    # Create the optimizer\n",
        "    optimizer = AdamW(bert_classifier.parameters(),\n",
        "                      lr=2e-5,    # Default learning rate\n",
        "                      eps=1e-8    # Default epsilon value\n",
        "                      )\n",
        "\n",
        "    # Total number of training steps\n",
        "    total_steps = len(train_dataloader) * epochs\n",
        "\n",
        "    # Set up the learning rate scheduler\n",
        "    scheduler = get_linear_schedule_with_warmup(optimizer,\n",
        "                                                num_warmup_steps=0, # Default value\n",
        "                                                num_training_steps=total_steps)\n",
        "    return bert_classifier, optimizer, scheduler"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Clv6M7KvD_KX"
      },
      "source": [
        "import random\n",
        "import time\n",
        "\n",
        "# Specify loss function\n",
        "loss_fn = nn.CrossEntropyLoss()\n",
        "\n",
        "def set_seed(seed_value=42):\n",
        "    \"\"\"Set seed for reproducibility.\n",
        "    \"\"\"\n",
        "    random.seed(seed_value)\n",
        "    np.random.seed(seed_value)\n",
        "    torch.manual_seed(seed_value)\n",
        "    torch.cuda.manual_seed_all(seed_value)\n",
        "\n",
        "def train(model, train_dataloader, val_dataloader=None, epochs=4, evaluation=False):\n",
        "    \"\"\"Train the BertClassifier model.\n",
        "    \"\"\"\n",
        "    # Start training loop\n",
        "    print(\"Start training...\\n\")\n",
        "    for epoch_i in range(epochs):\n",
        "        # =======================================\n",
        "        #               Training\n",
        "        # =======================================\n",
        "        # Print the header of the result table\n",
        "        print(f\"{'Epoch':^7} | {'Batch':^7} | {'Train Loss':^12} | {'Val Loss':^10} | {'Val Acc':^9} | {'Elapsed':^9}\")\n",
        "        print(\"-\"*70)\n",
        "\n",
        "        # Measure the elapsed time of each epoch\n",
        "        t0_epoch, t0_batch = time.time(), time.time()\n",
        "\n",
        "        # Reset tracking variables at the beginning of each epoch\n",
        "        total_loss, batch_loss, batch_counts = 0, 0, 0\n",
        "\n",
        "        # Put the model into the training mode\n",
        "        model.train()\n",
        "\n",
        "        # For each batch of training data...\n",
        "        for step, batch in enumerate(train_dataloader):\n",
        "            batch_counts +=1\n",
        "            # Load batch to GPU\n",
        "            b_input_ids, b_attn_mask, b_labels = tuple(t.to(device) for t in batch)\n",
        "\n",
        "            # Zero out any previously calculated gradients\n",
        "            model.zero_grad()\n",
        "\n",
        "            # Perform a forward pass. This will return logits.\n",
        "            logits = model(b_input_ids, b_attn_mask)\n",
        "\n",
        "            # Compute loss and accumulate the loss values\n",
        "            loss = loss_fn(logits, b_labels)\n",
        "            batch_loss += loss.item()\n",
        "            total_loss += loss.item()\n",
        "\n",
        "            # Perform a backward pass to calculate gradients\n",
        "            loss.backward()\n",
        "\n",
        "            # Clip the norm of the gradients to 1.0 to prevent \"exploding gradients\"\n",
        "            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "            # Update parameters and the learning rate\n",
        "            optimizer.step()\n",
        "            scheduler.step()\n",
        "\n",
        "            # Print the loss values and time elapsed for every 20 batches\n",
        "            if (step % 20 == 0 and step != 0) or (step == len(train_dataloader) - 1):\n",
        "                # Calculate time elapsed for 20 batches\n",
        "                time_elapsed = time.time() - t0_batch\n",
        "\n",
        "                # Print training results\n",
        "                print(f\"{epoch_i + 1:^7} | {step:^7} | {batch_loss / batch_counts:^12.6f} | {'-':^10} | {'-':^9} | {time_elapsed:^9.2f}\")\n",
        "\n",
        "                # Reset batch tracking variables\n",
        "                batch_loss, batch_counts = 0, 0\n",
        "                t0_batch = time.time()\n",
        "\n",
        "        # Calculate the average loss over the entire training data\n",
        "        avg_train_loss = total_loss / len(train_dataloader)\n",
        "\n",
        "        print(\"-\"*70)\n",
        "        # =======================================\n",
        "        #               Evaluation\n",
        "        # =======================================\n",
        "        if evaluation == True:\n",
        "            # After the completion of each training epoch, measure the model's performance\n",
        "            # on our validation set.\n",
        "            val_loss, val_accuracy = evaluate(model, val_dataloader)\n",
        "\n",
        "            # Print performance over the entire training data\n",
        "            time_elapsed = time.time() - t0_epoch\n",
        "            \n",
        "            print(f\"{epoch_i + 1:^7} | {'-':^7} | {avg_train_loss:^12.6f} | {val_loss:^10.6f} | {val_accuracy:^9.2f} | {time_elapsed:^9.2f}\")\n",
        "            print(\"-\"*70)\n",
        "        print(\"\\n\")\n",
        "\n",
        "    \n",
        "    print(\"Training complete!\")\n",
        "    return float(val_loss), float(val_accuracy)\n",
        "\n",
        "def evaluate(model, val_dataloader):\n",
        "    \"\"\"After the completion of each training epoch, measure the model's performance\n",
        "    on our validation set.\n",
        "    \"\"\"\n",
        "    # Put the model into the evaluation mode. The dropout layers are disabled during\n",
        "    # the test time.\n",
        "    model.eval()\n",
        "\n",
        "    # Tracking variables\n",
        "    val_accuracy = []\n",
        "    val_loss = []\n",
        "\n",
        "    # For each batch in our validation set...\n",
        "    for batch in val_dataloader:\n",
        "        # Load batch to GPU\n",
        "        b_input_ids, b_attn_mask, b_labels = tuple(t.to(device) for t in batch)\n",
        "\n",
        "        # Compute logits\n",
        "        with torch.no_grad():\n",
        "            logits = model(b_input_ids, b_attn_mask)\n",
        "\n",
        "        # Compute loss\n",
        "        loss = loss_fn(logits, b_labels)\n",
        "        val_loss.append(loss.item())\n",
        "\n",
        "        # Get the predictions\n",
        "        preds = torch.argmax(logits, dim=1).flatten()\n",
        "\n",
        "        # Calculate the accuracy rate\n",
        "        accuracy = (preds == b_labels).cpu().numpy().mean() * 100\n",
        "        val_accuracy.append(accuracy)\n",
        "\n",
        "    # Compute the average accuracy and loss over the validation set.\n",
        "    val_loss = np.mean(val_loss)\n",
        "    val_accuracy = np.mean(val_accuracy)\n",
        "\n",
        "    return val_loss, val_accuracy"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Yr8-0_T1QnjF"
      },
      "source": [
        "## 3. Random Sampling <a class=\"anchor\" id=\"section_3\"></a>\n",
        "\n",
        "We have equally collect 50,000 articles from peaceful and 50,000 articles from non-peaceful countries."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EGJut4cyJirH",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3c174b56-75cd-4d83-e357-2418bdb697dd"
      },
      "source": [
        "import random\n",
        "\n",
        "peace_index = random.sample(list(df_whole[df_whole['label'] == 0].index), 50000)\n",
        "nonpeaceful_index = random.sample(list(df_whole[df_whole['label'] == 1].index), 50000)\n",
        "index = peace_index + nonpeaceful_index\n",
        "\n",
        "X = df_whole['text'][index]\n",
        "y1 = df_whole['label'][index]\n",
        "X.index = np.arange(0,len(X))\n",
        "y1.index = np.arange(0,len(y1))\n",
        "\n",
        "inputs, masks = preprocessing_for_bert(X)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/transformers/tokenization_utils_base.py:2022: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HoNcCkF7EA8w",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "d01b0aa7b40048af93a25bdafcf56f5b",
            "0aeb351cfc91401ebbd8d83307951554",
            "136b6bf92f9f4da2809c81d222f93c8c",
            "b5ddd4651eb049528265aadfaca4df41",
            "1dad64454cb0455f8255e730454bf00b",
            "b61560a3de7c4d43a2f97b3613d334e5",
            "8a15be73d30349398255c3e51f440d18",
            "5911b7dc0507455ebac902d25daf4981",
            "0b1fcf138b98437f95480d2ae106e8ac",
            "4df77559eeec49b29d983b714ac4347c",
            "5b3ad5fc37184afab8b752b58869e16c",
            "b0f84fc90b0a48c7beebb7843cbb7469",
            "51fb22e38c8d43f8a558833c25b40667",
            "38f9f520bf744db68fe1bf6549f2d1a7",
            "a1c0421189374d51adee47e0a98c240e",
            "1849991a582c4885bdd1e9c6dad04861"
          ]
        },
        "outputId": "494f8c94-5fe6-47b5-e340-a577b7cc9df2"
      },
      "source": [
        "from sklearn.model_selection import KFold\n",
        "#train and val are indices\n",
        "kf = KFold(n_splits=5, shuffle = True, random_state = 42)\n",
        "\n",
        "\n",
        "batch_size = 16\n",
        "train_loss = []\n",
        "val_loss = []\n",
        "val_accuracy = []\n",
        "\n",
        "for train_index, val_index in kf.split(inputs):\n",
        "  #Data Preparation\n",
        "  train_inputs = inputs[train_index]\n",
        "  train_masks = masks[train_index]\n",
        "  val_inputs = inputs[val_index]\n",
        "  val_masks = masks[val_index]\n",
        "\n",
        "  train_labels = torch.tensor(y1[train_index].values)\n",
        "  val_labels = torch.tensor(y1[val_index].values)\n",
        "  \n",
        "  #Data Loader Class\n",
        "  train_data = TensorDataset(train_inputs, train_masks, train_labels)\n",
        "  train_sampler = RandomSampler(train_data)\n",
        "  train_dataloader = DataLoader(train_data, sampler = train_sampler, batch_size = batch_size)\n",
        "\n",
        "  val_data = TensorDataset(val_inputs, val_masks, val_labels)\n",
        "  val_sampler = RandomSampler(val_data)\n",
        "  val_dataloader = DataLoader(val_data, sampler = val_sampler, batch_size = batch_size)\n",
        "\n",
        "  #Fine Tune and Evaluation\n",
        "  set_seed(42)    # Set seed for reproducibility\n",
        "  bert_classifier, optimizer, scheduler = initialize_model(epochs=2)\n",
        "  val_loss1, val_accuracy1 = train(bert_classifier, train_dataloader, val_dataloader, epochs=2, evaluation=True)\n",
        "  \n",
        "  val_loss.append(val_loss1)\n",
        "  val_accuracy.append(val_accuracy1)\n",
        "  break"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d01b0aa7b40048af93a25bdafcf56f5b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=433.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0b1fcf138b98437f95480d2ae106e8ac",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=440473133.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Start training...\n",
            "\n",
            " Epoch  |  Batch  |  Train Loss  |  Val Loss  |  Val Acc  |  Elapsed \n",
            "----------------------------------------------------------------------\n",
            "   1    |   20    |   0.584987   |     -      |     -     |   18.19  \n",
            "   1    |   40    |   0.369717   |     -      |     -     |   17.22  \n",
            "   1    |   60    |   0.333490   |     -      |     -     |   17.28  \n",
            "   1    |   80    |   0.277348   |     -      |     -     |   17.21  \n",
            "   1    |   100   |   0.283571   |     -      |     -     |   17.20  \n",
            "   1    |   120   |   0.153531   |     -      |     -     |   17.23  \n",
            "   1    |   140   |   0.274527   |     -      |     -     |   17.19  \n",
            "   1    |   160   |   0.319057   |     -      |     -     |   17.18  \n",
            "   1    |   180   |   0.223176   |     -      |     -     |   17.22  \n",
            "   1    |   200   |   0.251742   |     -      |     -     |   17.26  \n",
            "   1    |   220   |   0.246898   |     -      |     -     |   17.24  \n",
            "   1    |   240   |   0.286296   |     -      |     -     |   17.18  \n",
            "   1    |   260   |   0.240397   |     -      |     -     |   17.22  \n",
            "   1    |   280   |   0.269313   |     -      |     -     |   17.16  \n",
            "   1    |   300   |   0.270837   |     -      |     -     |   17.17  \n",
            "   1    |   320   |   0.294338   |     -      |     -     |   17.15  \n",
            "   1    |   340   |   0.203515   |     -      |     -     |   17.20  \n",
            "   1    |   360   |   0.241437   |     -      |     -     |   17.15  \n",
            "   1    |   380   |   0.234879   |     -      |     -     |   17.15  \n",
            "   1    |   400   |   0.196689   |     -      |     -     |   17.17  \n",
            "   1    |   420   |   0.238552   |     -      |     -     |   17.15  \n",
            "   1    |   440   |   0.246554   |     -      |     -     |   17.20  \n",
            "   1    |   460   |   0.257836   |     -      |     -     |   17.23  \n",
            "   1    |   480   |   0.237227   |     -      |     -     |   17.23  \n",
            "   1    |   500   |   0.214429   |     -      |     -     |   17.21  \n",
            "   1    |   520   |   0.238192   |     -      |     -     |   17.17  \n",
            "   1    |   540   |   0.249142   |     -      |     -     |   17.17  \n",
            "   1    |   560   |   0.189350   |     -      |     -     |   17.19  \n",
            "   1    |   580   |   0.177382   |     -      |     -     |   17.22  \n",
            "   1    |   600   |   0.221309   |     -      |     -     |   17.22  \n",
            "   1    |   620   |   0.252641   |     -      |     -     |   17.17  \n",
            "   1    |   640   |   0.240956   |     -      |     -     |   17.16  \n",
            "   1    |   660   |   0.206984   |     -      |     -     |   17.20  \n",
            "   1    |   680   |   0.194627   |     -      |     -     |   17.17  \n",
            "   1    |   700   |   0.202270   |     -      |     -     |   17.19  \n",
            "   1    |   720   |   0.230007   |     -      |     -     |   17.18  \n",
            "   1    |   740   |   0.186829   |     -      |     -     |   17.22  \n",
            "   1    |   760   |   0.263453   |     -      |     -     |   17.26  \n",
            "   1    |   780   |   0.185544   |     -      |     -     |   17.21  \n",
            "   1    |   800   |   0.202269   |     -      |     -     |   17.19  \n",
            "   1    |   820   |   0.199020   |     -      |     -     |   17.22  \n",
            "   1    |   840   |   0.219861   |     -      |     -     |   17.21  \n",
            "   1    |   860   |   0.246324   |     -      |     -     |   17.20  \n",
            "   1    |   880   |   0.218401   |     -      |     -     |   17.22  \n",
            "   1    |   900   |   0.255499   |     -      |     -     |   17.21  \n",
            "   1    |   920   |   0.195949   |     -      |     -     |   17.23  \n",
            "   1    |   940   |   0.278076   |     -      |     -     |   17.21  \n",
            "   1    |   960   |   0.173561   |     -      |     -     |   17.18  \n",
            "   1    |   980   |   0.240465   |     -      |     -     |   17.21  \n",
            "   1    |  1000   |   0.191998   |     -      |     -     |   17.25  \n",
            "   1    |  1020   |   0.181946   |     -      |     -     |   17.16  \n",
            "   1    |  1040   |   0.271787   |     -      |     -     |   17.20  \n",
            "   1    |  1060   |   0.214062   |     -      |     -     |   17.23  \n",
            "   1    |  1080   |   0.175532   |     -      |     -     |   17.24  \n",
            "   1    |  1100   |   0.193770   |     -      |     -     |   17.25  \n",
            "   1    |  1120   |   0.244741   |     -      |     -     |   17.31  \n",
            "   1    |  1140   |   0.241414   |     -      |     -     |   17.20  \n",
            "   1    |  1160   |   0.159917   |     -      |     -     |   17.22  \n",
            "   1    |  1180   |   0.170447   |     -      |     -     |   17.20  \n",
            "   1    |  1200   |   0.161692   |     -      |     -     |   17.18  \n",
            "   1    |  1220   |   0.202236   |     -      |     -     |   17.19  \n",
            "   1    |  1240   |   0.266674   |     -      |     -     |   17.23  \n",
            "   1    |  1260   |   0.195828   |     -      |     -     |   17.23  \n",
            "   1    |  1280   |   0.164500   |     -      |     -     |   17.26  \n",
            "   1    |  1300   |   0.152674   |     -      |     -     |   17.21  \n",
            "   1    |  1320   |   0.236077   |     -      |     -     |   17.18  \n",
            "   1    |  1340   |   0.206345   |     -      |     -     |   17.18  \n",
            "   1    |  1360   |   0.174600   |     -      |     -     |   17.17  \n",
            "   1    |  1380   |   0.168560   |     -      |     -     |   17.18  \n",
            "   1    |  1400   |   0.215045   |     -      |     -     |   17.17  \n",
            "   1    |  1420   |   0.197794   |     -      |     -     |   17.17  \n",
            "   1    |  1440   |   0.168478   |     -      |     -     |   17.20  \n",
            "   1    |  1460   |   0.191415   |     -      |     -     |   17.26  \n",
            "   1    |  1480   |   0.221171   |     -      |     -     |   17.20  \n",
            "   1    |  1500   |   0.186555   |     -      |     -     |   17.21  \n",
            "   1    |  1520   |   0.166340   |     -      |     -     |   17.19  \n",
            "   1    |  1540   |   0.170639   |     -      |     -     |   17.25  \n",
            "   1    |  1560   |   0.163735   |     -      |     -     |   17.23  \n",
            "   1    |  1580   |   0.188049   |     -      |     -     |   17.23  \n",
            "   1    |  1600   |   0.213043   |     -      |     -     |   17.23  \n",
            "   1    |  1620   |   0.216320   |     -      |     -     |   17.24  \n",
            "   1    |  1640   |   0.156862   |     -      |     -     |   17.23  \n",
            "   1    |  1660   |   0.179302   |     -      |     -     |   17.22  \n",
            "   1    |  1680   |   0.219995   |     -      |     -     |   17.16  \n",
            "   1    |  1700   |   0.205990   |     -      |     -     |   17.19  \n",
            "   1    |  1720   |   0.141910   |     -      |     -     |   17.18  \n",
            "   1    |  1740   |   0.177367   |     -      |     -     |   17.18  \n",
            "   1    |  1760   |   0.184148   |     -      |     -     |   17.18  \n",
            "   1    |  1780   |   0.201380   |     -      |     -     |   17.18  \n",
            "   1    |  1800   |   0.197172   |     -      |     -     |   17.16  \n",
            "   1    |  1820   |   0.161719   |     -      |     -     |   17.17  \n",
            "   1    |  1840   |   0.159360   |     -      |     -     |   17.19  \n",
            "   1    |  1860   |   0.213396   |     -      |     -     |   17.18  \n",
            "   1    |  1880   |   0.170731   |     -      |     -     |   17.20  \n",
            "   1    |  1900   |   0.105984   |     -      |     -     |   17.20  \n",
            "   1    |  1920   |   0.215321   |     -      |     -     |   17.22  \n",
            "   1    |  1940   |   0.214385   |     -      |     -     |   17.18  \n",
            "   1    |  1960   |   0.120295   |     -      |     -     |   17.17  \n",
            "   1    |  1980   |   0.292898   |     -      |     -     |   17.17  \n",
            "   1    |  2000   |   0.182279   |     -      |     -     |   17.21  \n",
            "   1    |  2020   |   0.247646   |     -      |     -     |   17.18  \n",
            "   1    |  2040   |   0.218362   |     -      |     -     |   17.22  \n",
            "   1    |  2060   |   0.217505   |     -      |     -     |   17.19  \n",
            "   1    |  2080   |   0.224517   |     -      |     -     |   17.22  \n",
            "   1    |  2100   |   0.175296   |     -      |     -     |   17.23  \n",
            "   1    |  2120   |   0.164117   |     -      |     -     |   17.26  \n",
            "   1    |  2140   |   0.201651   |     -      |     -     |   17.30  \n",
            "   1    |  2160   |   0.230717   |     -      |     -     |   17.15  \n",
            "   1    |  2180   |   0.135447   |     -      |     -     |   17.21  \n",
            "   1    |  2200   |   0.118593   |     -      |     -     |   17.22  \n",
            "   1    |  2220   |   0.195288   |     -      |     -     |   17.21  \n",
            "   1    |  2240   |   0.226459   |     -      |     -     |   17.22  \n",
            "   1    |  2260   |   0.149635   |     -      |     -     |   17.21  \n",
            "   1    |  2280   |   0.199814   |     -      |     -     |   17.21  \n",
            "   1    |  2300   |   0.171570   |     -      |     -     |   17.19  \n",
            "   1    |  2320   |   0.170443   |     -      |     -     |   17.19  \n",
            "   1    |  2340   |   0.128950   |     -      |     -     |   17.16  \n",
            "   1    |  2360   |   0.129945   |     -      |     -     |   17.18  \n",
            "   1    |  2380   |   0.265446   |     -      |     -     |   17.18  \n",
            "   1    |  2400   |   0.228255   |     -      |     -     |   17.20  \n",
            "   1    |  2420   |   0.218336   |     -      |     -     |   17.19  \n",
            "   1    |  2440   |   0.179297   |     -      |     -     |   17.17  \n",
            "   1    |  2460   |   0.177160   |     -      |     -     |   17.19  \n",
            "   1    |  2480   |   0.245429   |     -      |     -     |   17.18  \n",
            "   1    |  2500   |   0.162696   |     -      |     -     |   17.22  \n",
            "   1    |  2520   |   0.227147   |     -      |     -     |   17.19  \n",
            "   1    |  2540   |   0.213602   |     -      |     -     |   17.29  \n",
            "   1    |  2560   |   0.200455   |     -      |     -     |   17.25  \n",
            "   1    |  2580   |   0.198356   |     -      |     -     |   17.17  \n",
            "   1    |  2600   |   0.113770   |     -      |     -     |   17.21  \n",
            "   1    |  2620   |   0.146601   |     -      |     -     |   17.19  \n",
            "   1    |  2640   |   0.119220   |     -      |     -     |   17.20  \n",
            "   1    |  2660   |   0.198211   |     -      |     -     |   17.16  \n",
            "   1    |  2680   |   0.108505   |     -      |     -     |   17.15  \n",
            "   1    |  2700   |   0.196408   |     -      |     -     |   17.18  \n",
            "   1    |  2720   |   0.129817   |     -      |     -     |   17.23  \n",
            "   1    |  2740   |   0.173967   |     -      |     -     |   17.18  \n",
            "   1    |  2760   |   0.208635   |     -      |     -     |   17.19  \n",
            "   1    |  2780   |   0.160445   |     -      |     -     |   17.22  \n",
            "   1    |  2800   |   0.176161   |     -      |     -     |   17.22  \n",
            "   1    |  2820   |   0.203404   |     -      |     -     |   17.20  \n",
            "   1    |  2840   |   0.164475   |     -      |     -     |   17.17  \n",
            "   1    |  2860   |   0.174728   |     -      |     -     |   17.25  \n",
            "   1    |  2880   |   0.177436   |     -      |     -     |   17.27  \n",
            "   1    |  2900   |   0.133467   |     -      |     -     |   17.23  \n",
            "   1    |  2920   |   0.196408   |     -      |     -     |   17.21  \n",
            "   1    |  2940   |   0.262187   |     -      |     -     |   17.21  \n",
            "   1    |  2960   |   0.198980   |     -      |     -     |   17.23  \n",
            "   1    |  2980   |   0.127014   |     -      |     -     |   17.23  \n",
            "   1    |  3000   |   0.163111   |     -      |     -     |   17.26  \n",
            "   1    |  3020   |   0.215240   |     -      |     -     |   17.25  \n",
            "   1    |  3040   |   0.159725   |     -      |     -     |   17.27  \n",
            "   1    |  3060   |   0.129078   |     -      |     -     |   17.25  \n",
            "   1    |  3080   |   0.193743   |     -      |     -     |   17.27  \n",
            "   1    |  3100   |   0.199652   |     -      |     -     |   17.23  \n",
            "   1    |  3120   |   0.206097   |     -      |     -     |   17.25  \n",
            "   1    |  3140   |   0.147797   |     -      |     -     |   17.25  \n",
            "   1    |  3160   |   0.131381   |     -      |     -     |   17.26  \n",
            "   1    |  3180   |   0.189307   |     -      |     -     |   17.22  \n",
            "   1    |  3200   |   0.144296   |     -      |     -     |   17.17  \n",
            "   1    |  3220   |   0.219562   |     -      |     -     |   17.22  \n",
            "   1    |  3240   |   0.174836   |     -      |     -     |   17.21  \n",
            "   1    |  3260   |   0.197042   |     -      |     -     |   17.21  \n",
            "   1    |  3280   |   0.177235   |     -      |     -     |   17.25  \n",
            "   1    |  3300   |   0.173665   |     -      |     -     |   17.25  \n",
            "   1    |  3320   |   0.152518   |     -      |     -     |   17.22  \n",
            "   1    |  3340   |   0.147473   |     -      |     -     |   17.24  \n",
            "   1    |  3360   |   0.144507   |     -      |     -     |   17.19  \n",
            "   1    |  3380   |   0.181316   |     -      |     -     |   17.20  \n",
            "   1    |  3400   |   0.200768   |     -      |     -     |   17.18  \n",
            "   1    |  3420   |   0.173802   |     -      |     -     |   17.21  \n",
            "   1    |  3440   |   0.184182   |     -      |     -     |   17.24  \n",
            "   1    |  3460   |   0.161252   |     -      |     -     |   17.21  \n",
            "   1    |  3480   |   0.198644   |     -      |     -     |   17.22  \n",
            "   1    |  3500   |   0.143576   |     -      |     -     |   17.22  \n",
            "   1    |  3520   |   0.177738   |     -      |     -     |   17.18  \n",
            "   1    |  3540   |   0.166059   |     -      |     -     |   17.21  \n",
            "   1    |  3560   |   0.172142   |     -      |     -     |   17.19  \n",
            "   1    |  3580   |   0.193112   |     -      |     -     |   17.19  \n",
            "   1    |  3600   |   0.204638   |     -      |     -     |   17.24  \n",
            "   1    |  3620   |   0.154391   |     -      |     -     |   17.22  \n",
            "   1    |  3640   |   0.118258   |     -      |     -     |   17.21  \n",
            "   1    |  3660   |   0.121646   |     -      |     -     |   17.20  \n",
            "   1    |  3680   |   0.143988   |     -      |     -     |   17.21  \n",
            "   1    |  3700   |   0.121308   |     -      |     -     |   17.19  \n",
            "   1    |  3720   |   0.164938   |     -      |     -     |   17.24  \n",
            "   1    |  3740   |   0.164024   |     -      |     -     |   17.18  \n",
            "   1    |  3760   |   0.116198   |     -      |     -     |   17.17  \n",
            "   1    |  3780   |   0.207486   |     -      |     -     |   17.27  \n",
            "   1    |  3800   |   0.164306   |     -      |     -     |   17.26  \n",
            "   1    |  3820   |   0.199821   |     -      |     -     |   17.28  \n",
            "   1    |  3840   |   0.117587   |     -      |     -     |   17.21  \n",
            "   1    |  3860   |   0.134568   |     -      |     -     |   17.24  \n",
            "   1    |  3880   |   0.165202   |     -      |     -     |   17.22  \n",
            "   1    |  3900   |   0.199030   |     -      |     -     |   17.24  \n",
            "   1    |  3920   |   0.215349   |     -      |     -     |   17.23  \n",
            "   1    |  3940   |   0.127231   |     -      |     -     |   17.23  \n",
            "   1    |  3960   |   0.165905   |     -      |     -     |   17.21  \n",
            "   1    |  3980   |   0.218917   |     -      |     -     |   17.25  \n",
            "   1    |  4000   |   0.191521   |     -      |     -     |   17.23  \n",
            "   1    |  4020   |   0.145467   |     -      |     -     |   17.25  \n",
            "   1    |  4040   |   0.160740   |     -      |     -     |   17.22  \n",
            "   1    |  4060   |   0.179812   |     -      |     -     |   17.29  \n",
            "   1    |  4080   |   0.132509   |     -      |     -     |   17.20  \n",
            "   1    |  4100   |   0.186353   |     -      |     -     |   17.19  \n",
            "   1    |  4120   |   0.115526   |     -      |     -     |   17.20  \n",
            "   1    |  4140   |   0.165875   |     -      |     -     |   17.20  \n",
            "   1    |  4160   |   0.180874   |     -      |     -     |   17.22  \n",
            "   1    |  4180   |   0.144687   |     -      |     -     |   17.22  \n",
            "   1    |  4200   |   0.134432   |     -      |     -     |   17.24  \n",
            "   1    |  4220   |   0.144288   |     -      |     -     |   17.23  \n",
            "   1    |  4240   |   0.157081   |     -      |     -     |   17.26  \n",
            "   1    |  4260   |   0.264626   |     -      |     -     |   17.24  \n",
            "   1    |  4280   |   0.183957   |     -      |     -     |   17.22  \n",
            "   1    |  4300   |   0.097536   |     -      |     -     |   17.20  \n",
            "   1    |  4320   |   0.174761   |     -      |     -     |   17.27  \n",
            "   1    |  4340   |   0.255658   |     -      |     -     |   17.27  \n",
            "   1    |  4360   |   0.159066   |     -      |     -     |   17.29  \n",
            "   1    |  4380   |   0.195722   |     -      |     -     |   17.25  \n",
            "   1    |  4400   |   0.155219   |     -      |     -     |   17.27  \n",
            "   1    |  4420   |   0.210028   |     -      |     -     |   17.20  \n",
            "   1    |  4440   |   0.161946   |     -      |     -     |   17.23  \n",
            "   1    |  4460   |   0.139619   |     -      |     -     |   17.20  \n",
            "   1    |  4480   |   0.171041   |     -      |     -     |   17.22  \n",
            "   1    |  4500   |   0.148768   |     -      |     -     |   17.25  \n",
            "   1    |  4520   |   0.185834   |     -      |     -     |   17.23  \n",
            "   1    |  4540   |   0.168986   |     -      |     -     |   17.29  \n",
            "   1    |  4560   |   0.162377   |     -      |     -     |   17.20  \n",
            "   1    |  4580   |   0.145464   |     -      |     -     |   17.21  \n",
            "   1    |  4600   |   0.148243   |     -      |     -     |   17.23  \n",
            "   1    |  4620   |   0.174054   |     -      |     -     |   17.26  \n",
            "   1    |  4640   |   0.131982   |     -      |     -     |   17.22  \n",
            "   1    |  4660   |   0.174331   |     -      |     -     |   17.18  \n",
            "   1    |  4680   |   0.183396   |     -      |     -     |   17.23  \n",
            "   1    |  4700   |   0.215029   |     -      |     -     |   17.24  \n",
            "   1    |  4720   |   0.122079   |     -      |     -     |   17.22  \n",
            "   1    |  4740   |   0.123088   |     -      |     -     |   17.29  \n",
            "   1    |  4760   |   0.162865   |     -      |     -     |   17.26  \n",
            "   1    |  4780   |   0.153428   |     -      |     -     |   17.29  \n",
            "   1    |  4800   |   0.135693   |     -      |     -     |   17.26  \n",
            "   1    |  4820   |   0.161085   |     -      |     -     |   17.30  \n",
            "   1    |  4840   |   0.095953   |     -      |     -     |   17.23  \n",
            "   1    |  4860   |   0.179270   |     -      |     -     |   17.27  \n",
            "   1    |  4880   |   0.188362   |     -      |     -     |   17.22  \n",
            "   1    |  4900   |   0.119616   |     -      |     -     |   17.28  \n",
            "   1    |  4920   |   0.161251   |     -      |     -     |   17.18  \n",
            "   1    |  4940   |   0.129416   |     -      |     -     |   17.23  \n",
            "   1    |  4960   |   0.191993   |     -      |     -     |   17.26  \n",
            "   1    |  4980   |   0.183285   |     -      |     -     |   17.25  \n",
            "   1    |  4999   |   0.090839   |     -      |     -     |   16.37  \n",
            "----------------------------------------------------------------------\n",
            "   1    |    -    |   0.189991   |  0.149268  |   94.25   |  4661.05 \n",
            "----------------------------------------------------------------------\n",
            "\n",
            "\n",
            " Epoch  |  Batch  |  Train Loss  |  Val Loss  |  Val Acc  |  Elapsed \n",
            "----------------------------------------------------------------------\n",
            "   2    |   20    |   0.102994   |     -      |     -     |   18.15  \n",
            "   2    |   40    |   0.085307   |     -      |     -     |   17.29  \n",
            "   2    |   60    |   0.071899   |     -      |     -     |   17.25  \n",
            "   2    |   80    |   0.088618   |     -      |     -     |   17.25  \n",
            "   2    |   100   |   0.118072   |     -      |     -     |   17.21  \n",
            "   2    |   120   |   0.129166   |     -      |     -     |   17.30  \n",
            "   2    |   140   |   0.118862   |     -      |     -     |   17.23  \n",
            "   2    |   160   |   0.144834   |     -      |     -     |   17.22  \n",
            "   2    |   180   |   0.136601   |     -      |     -     |   17.26  \n",
            "   2    |   200   |   0.091677   |     -      |     -     |   17.25  \n",
            "   2    |   220   |   0.116930   |     -      |     -     |   17.26  \n",
            "   2    |   240   |   0.090649   |     -      |     -     |   17.26  \n",
            "   2    |   260   |   0.084708   |     -      |     -     |   17.23  \n",
            "   2    |   280   |   0.080087   |     -      |     -     |   17.26  \n",
            "   2    |   300   |   0.173571   |     -      |     -     |   17.29  \n",
            "   2    |   320   |   0.054615   |     -      |     -     |   17.28  \n",
            "   2    |   340   |   0.139447   |     -      |     -     |   17.28  \n",
            "   2    |   360   |   0.171208   |     -      |     -     |   17.31  \n",
            "   2    |   380   |   0.124444   |     -      |     -     |   17.23  \n",
            "   2    |   400   |   0.143573   |     -      |     -     |   17.23  \n",
            "   2    |   420   |   0.105626   |     -      |     -     |   17.25  \n",
            "   2    |   440   |   0.074868   |     -      |     -     |   17.16  \n",
            "   2    |   460   |   0.104351   |     -      |     -     |   17.20  \n",
            "   2    |   480   |   0.115932   |     -      |     -     |   17.26  \n",
            "   2    |   500   |   0.109998   |     -      |     -     |   17.25  \n",
            "   2    |   520   |   0.049760   |     -      |     -     |   17.24  \n",
            "   2    |   540   |   0.098512   |     -      |     -     |   17.27  \n",
            "   2    |   560   |   0.179440   |     -      |     -     |   17.23  \n",
            "   2    |   580   |   0.091340   |     -      |     -     |   17.26  \n",
            "   2    |   600   |   0.071013   |     -      |     -     |   17.27  \n",
            "   2    |   620   |   0.138895   |     -      |     -     |   17.19  \n",
            "   2    |   640   |   0.166467   |     -      |     -     |   17.18  \n",
            "   2    |   660   |   0.105858   |     -      |     -     |   17.19  \n",
            "   2    |   680   |   0.133865   |     -      |     -     |   17.20  \n",
            "   2    |   700   |   0.109870   |     -      |     -     |   17.21  \n",
            "   2    |   720   |   0.131628   |     -      |     -     |   17.25  \n",
            "   2    |   740   |   0.089698   |     -      |     -     |   17.23  \n",
            "   2    |   760   |   0.173638   |     -      |     -     |   17.30  \n",
            "   2    |   780   |   0.102522   |     -      |     -     |   17.25  \n",
            "   2    |   800   |   0.124964   |     -      |     -     |   17.24  \n",
            "   2    |   820   |   0.113263   |     -      |     -     |   17.27  \n",
            "   2    |   840   |   0.105897   |     -      |     -     |   17.32  \n",
            "   2    |   860   |   0.109478   |     -      |     -     |   17.26  \n",
            "   2    |   880   |   0.099100   |     -      |     -     |   17.18  \n",
            "   2    |   900   |   0.117684   |     -      |     -     |   17.20  \n",
            "   2    |   920   |   0.094040   |     -      |     -     |   17.23  \n",
            "   2    |   940   |   0.127619   |     -      |     -     |   17.19  \n",
            "   2    |   960   |   0.143021   |     -      |     -     |   17.24  \n",
            "   2    |   980   |   0.127575   |     -      |     -     |   17.28  \n",
            "   2    |  1000   |   0.106610   |     -      |     -     |   17.16  \n",
            "   2    |  1020   |   0.039500   |     -      |     -     |   17.19  \n",
            "   2    |  1040   |   0.115715   |     -      |     -     |   17.26  \n",
            "   2    |  1060   |   0.081220   |     -      |     -     |   17.25  \n",
            "   2    |  1080   |   0.086152   |     -      |     -     |   17.21  \n",
            "   2    |  1100   |   0.132585   |     -      |     -     |   17.19  \n",
            "   2    |  1120   |   0.097043   |     -      |     -     |   17.23  \n",
            "   2    |  1140   |   0.104250   |     -      |     -     |   17.22  \n",
            "   2    |  1160   |   0.153083   |     -      |     -     |   17.22  \n",
            "   2    |  1180   |   0.122880   |     -      |     -     |   17.18  \n",
            "   2    |  1200   |   0.114740   |     -      |     -     |   17.23  \n",
            "   2    |  1220   |   0.082044   |     -      |     -     |   17.20  \n",
            "   2    |  1240   |   0.124298   |     -      |     -     |   17.20  \n",
            "   2    |  1260   |   0.090530   |     -      |     -     |   17.20  \n",
            "   2    |  1280   |   0.092679   |     -      |     -     |   17.19  \n",
            "   2    |  1300   |   0.065728   |     -      |     -     |   17.19  \n",
            "   2    |  1320   |   0.126691   |     -      |     -     |   17.18  \n",
            "   2    |  1340   |   0.193481   |     -      |     -     |   17.19  \n",
            "   2    |  1360   |   0.124447   |     -      |     -     |   17.20  \n",
            "   2    |  1380   |   0.109377   |     -      |     -     |   17.18  \n",
            "   2    |  1400   |   0.098311   |     -      |     -     |   17.24  \n",
            "   2    |  1420   |   0.158594   |     -      |     -     |   17.24  \n",
            "   2    |  1440   |   0.107673   |     -      |     -     |   17.22  \n",
            "   2    |  1460   |   0.187877   |     -      |     -     |   17.26  \n",
            "   2    |  1480   |   0.088870   |     -      |     -     |   17.21  \n",
            "   2    |  1500   |   0.149230   |     -      |     -     |   17.26  \n",
            "   2    |  1520   |   0.099136   |     -      |     -     |   17.27  \n",
            "   2    |  1540   |   0.085659   |     -      |     -     |   17.18  \n",
            "   2    |  1560   |   0.123830   |     -      |     -     |   17.21  \n",
            "   2    |  1580   |   0.105309   |     -      |     -     |   17.20  \n",
            "   2    |  1600   |   0.165646   |     -      |     -     |   17.22  \n",
            "   2    |  1620   |   0.109335   |     -      |     -     |   17.19  \n",
            "   2    |  1640   |   0.123511   |     -      |     -     |   17.21  \n",
            "   2    |  1660   |   0.108423   |     -      |     -     |   17.24  \n",
            "   2    |  1680   |   0.117913   |     -      |     -     |   17.28  \n",
            "   2    |  1700   |   0.141384   |     -      |     -     |   17.24  \n",
            "   2    |  1720   |   0.128860   |     -      |     -     |   17.20  \n",
            "   2    |  1740   |   0.135211   |     -      |     -     |   17.19  \n",
            "   2    |  1760   |   0.112463   |     -      |     -     |   17.18  \n",
            "   2    |  1780   |   0.125151   |     -      |     -     |   17.20  \n",
            "   2    |  1800   |   0.143847   |     -      |     -     |   17.20  \n",
            "   2    |  1820   |   0.137824   |     -      |     -     |   17.24  \n",
            "   2    |  1840   |   0.120044   |     -      |     -     |   17.24  \n",
            "   2    |  1860   |   0.108152   |     -      |     -     |   17.22  \n",
            "   2    |  1880   |   0.100725   |     -      |     -     |   17.14  \n",
            "   2    |  1900   |   0.132186   |     -      |     -     |   17.23  \n",
            "   2    |  1920   |   0.095791   |     -      |     -     |   17.17  \n",
            "   2    |  1940   |   0.100437   |     -      |     -     |   17.21  \n",
            "   2    |  1960   |   0.176403   |     -      |     -     |   17.24  \n",
            "   2    |  1980   |   0.161124   |     -      |     -     |   17.21  \n",
            "   2    |  2000   |   0.088090   |     -      |     -     |   17.23  \n",
            "   2    |  2020   |   0.080365   |     -      |     -     |   17.19  \n",
            "   2    |  2040   |   0.134827   |     -      |     -     |   17.19  \n",
            "   2    |  2060   |   0.075953   |     -      |     -     |   17.22  \n",
            "   2    |  2080   |   0.078778   |     -      |     -     |   17.19  \n",
            "   2    |  2100   |   0.144950   |     -      |     -     |   17.21  \n",
            "   2    |  2120   |   0.052762   |     -      |     -     |   17.21  \n",
            "   2    |  2140   |   0.170055   |     -      |     -     |   17.23  \n",
            "   2    |  2160   |   0.129966   |     -      |     -     |   17.29  \n",
            "   2    |  2180   |   0.182536   |     -      |     -     |   17.28  \n",
            "   2    |  2200   |   0.144249   |     -      |     -     |   17.23  \n",
            "   2    |  2220   |   0.153669   |     -      |     -     |   17.29  \n",
            "   2    |  2240   |   0.124204   |     -      |     -     |   17.30  \n",
            "   2    |  2260   |   0.149112   |     -      |     -     |   17.21  \n",
            "   2    |  2280   |   0.116675   |     -      |     -     |   17.21  \n",
            "   2    |  2300   |   0.111161   |     -      |     -     |   17.19  \n",
            "   2    |  2320   |   0.130026   |     -      |     -     |   17.20  \n",
            "   2    |  2340   |   0.099208   |     -      |     -     |   17.20  \n",
            "   2    |  2360   |   0.118876   |     -      |     -     |   17.21  \n",
            "   2    |  2380   |   0.087129   |     -      |     -     |   17.28  \n",
            "   2    |  2400   |   0.089192   |     -      |     -     |   17.25  \n",
            "   2    |  2420   |   0.133582   |     -      |     -     |   17.27  \n",
            "   2    |  2440   |   0.114365   |     -      |     -     |   17.30  \n",
            "   2    |  2460   |   0.113579   |     -      |     -     |   17.29  \n",
            "   2    |  2480   |   0.100840   |     -      |     -     |   17.21  \n",
            "   2    |  2500   |   0.106144   |     -      |     -     |   17.20  \n",
            "   2    |  2520   |   0.075802   |     -      |     -     |   17.19  \n",
            "   2    |  2540   |   0.132717   |     -      |     -     |   17.19  \n",
            "   2    |  2560   |   0.114878   |     -      |     -     |   17.20  \n",
            "   2    |  2580   |   0.057583   |     -      |     -     |   17.20  \n",
            "   2    |  2600   |   0.058829   |     -      |     -     |   17.18  \n",
            "   2    |  2620   |   0.173518   |     -      |     -     |   17.21  \n",
            "   2    |  2640   |   0.125511   |     -      |     -     |   17.17  \n",
            "   2    |  2660   |   0.140553   |     -      |     -     |   17.16  \n",
            "   2    |  2680   |   0.125431   |     -      |     -     |   17.16  \n",
            "   2    |  2700   |   0.077145   |     -      |     -     |   17.19  \n",
            "   2    |  2720   |   0.103981   |     -      |     -     |   17.17  \n",
            "   2    |  2740   |   0.116612   |     -      |     -     |   17.21  \n",
            "   2    |  2760   |   0.077666   |     -      |     -     |   17.19  \n",
            "   2    |  2780   |   0.101649   |     -      |     -     |   17.22  \n",
            "   2    |  2800   |   0.103707   |     -      |     -     |   17.22  \n",
            "   2    |  2820   |   0.136052   |     -      |     -     |   17.21  \n",
            "   2    |  2840   |   0.102196   |     -      |     -     |   17.20  \n",
            "   2    |  2860   |   0.083120   |     -      |     -     |   17.20  \n",
            "   2    |  2880   |   0.095820   |     -      |     -     |   17.20  \n",
            "   2    |  2900   |   0.092494   |     -      |     -     |   17.26  \n",
            "   2    |  2920   |   0.156234   |     -      |     -     |   17.26  \n",
            "   2    |  2940   |   0.063536   |     -      |     -     |   17.25  \n",
            "   2    |  2960   |   0.132234   |     -      |     -     |   17.22  \n",
            "   2    |  2980   |   0.161663   |     -      |     -     |   17.25  \n",
            "   2    |  3000   |   0.099373   |     -      |     -     |   17.30  \n",
            "   2    |  3020   |   0.100454   |     -      |     -     |   17.26  \n",
            "   2    |  3040   |   0.067509   |     -      |     -     |   17.21  \n",
            "   2    |  3060   |   0.090206   |     -      |     -     |   17.22  \n",
            "   2    |  3080   |   0.090784   |     -      |     -     |   17.19  \n",
            "   2    |  3100   |   0.142760   |     -      |     -     |   17.20  \n",
            "   2    |  3120   |   0.097477   |     -      |     -     |   17.18  \n",
            "   2    |  3140   |   0.129895   |     -      |     -     |   17.20  \n",
            "   2    |  3160   |   0.134587   |     -      |     -     |   17.20  \n",
            "   2    |  3180   |   0.134025   |     -      |     -     |   17.21  \n",
            "   2    |  3200   |   0.071090   |     -      |     -     |   17.16  \n",
            "   2    |  3220   |   0.088456   |     -      |     -     |   17.17  \n",
            "   2    |  3240   |   0.120192   |     -      |     -     |   17.21  \n",
            "   2    |  3260   |   0.151279   |     -      |     -     |   17.24  \n",
            "   2    |  3280   |   0.219637   |     -      |     -     |   17.24  \n",
            "   2    |  3300   |   0.124780   |     -      |     -     |   17.20  \n",
            "   2    |  3320   |   0.112600   |     -      |     -     |   17.21  \n",
            "   2    |  3340   |   0.122180   |     -      |     -     |   17.22  \n",
            "   2    |  3360   |   0.097524   |     -      |     -     |   17.20  \n",
            "   2    |  3380   |   0.147326   |     -      |     -     |   17.18  \n",
            "   2    |  3400   |   0.108975   |     -      |     -     |   17.19  \n",
            "   2    |  3420   |   0.071824   |     -      |     -     |   17.21  \n",
            "   2    |  3440   |   0.127703   |     -      |     -     |   17.24  \n",
            "   2    |  3460   |   0.066167   |     -      |     -     |   17.21  \n",
            "   2    |  3480   |   0.129173   |     -      |     -     |   17.19  \n",
            "   2    |  3500   |   0.123795   |     -      |     -     |   17.23  \n",
            "   2    |  3520   |   0.077419   |     -      |     -     |   17.19  \n",
            "   2    |  3540   |   0.054872   |     -      |     -     |   17.26  \n",
            "   2    |  3560   |   0.108564   |     -      |     -     |   17.24  \n",
            "   2    |  3580   |   0.105725   |     -      |     -     |   17.28  \n",
            "   2    |  3600   |   0.106971   |     -      |     -     |   17.26  \n",
            "   2    |  3620   |   0.115435   |     -      |     -     |   17.19  \n",
            "   2    |  3640   |   0.162660   |     -      |     -     |   17.33  \n",
            "   2    |  3660   |   0.154796   |     -      |     -     |   17.22  \n",
            "   2    |  3680   |   0.084799   |     -      |     -     |   17.21  \n",
            "   2    |  3700   |   0.064497   |     -      |     -     |   17.24  \n",
            "   2    |  3720   |   0.108105   |     -      |     -     |   17.16  \n",
            "   2    |  3740   |   0.116470   |     -      |     -     |   17.19  \n",
            "   2    |  3760   |   0.124511   |     -      |     -     |   17.19  \n",
            "   2    |  3780   |   0.141040   |     -      |     -     |   17.19  \n",
            "   2    |  3800   |   0.093219   |     -      |     -     |   17.19  \n",
            "   2    |  3820   |   0.074145   |     -      |     -     |   17.26  \n",
            "   2    |  3840   |   0.081410   |     -      |     -     |   17.24  \n",
            "   2    |  3860   |   0.087622   |     -      |     -     |   17.20  \n",
            "   2    |  3880   |   0.108043   |     -      |     -     |   17.23  \n",
            "   2    |  3900   |   0.052340   |     -      |     -     |   17.22  \n",
            "   2    |  3920   |   0.123033   |     -      |     -     |   17.28  \n",
            "   2    |  3940   |   0.068038   |     -      |     -     |   17.24  \n",
            "   2    |  3960   |   0.125049   |     -      |     -     |   17.23  \n",
            "   2    |  3980   |   0.127275   |     -      |     -     |   17.23  \n",
            "   2    |  4000   |   0.077064   |     -      |     -     |   17.28  \n",
            "   2    |  4020   |   0.184608   |     -      |     -     |   17.21  \n",
            "   2    |  4040   |   0.112036   |     -      |     -     |   17.25  \n",
            "   2    |  4060   |   0.073392   |     -      |     -     |   17.21  \n",
            "   2    |  4080   |   0.079488   |     -      |     -     |   17.21  \n",
            "   2    |  4100   |   0.133126   |     -      |     -     |   17.24  \n",
            "   2    |  4120   |   0.120376   |     -      |     -     |   17.26  \n",
            "   2    |  4140   |   0.093534   |     -      |     -     |   17.26  \n",
            "   2    |  4160   |   0.079326   |     -      |     -     |   17.20  \n",
            "   2    |  4180   |   0.134990   |     -      |     -     |   17.24  \n",
            "   2    |  4200   |   0.083145   |     -      |     -     |   17.20  \n",
            "   2    |  4220   |   0.094691   |     -      |     -     |   17.20  \n",
            "   2    |  4240   |   0.099602   |     -      |     -     |   17.19  \n",
            "   2    |  4260   |   0.076052   |     -      |     -     |   17.18  \n",
            "   2    |  4280   |   0.088115   |     -      |     -     |   17.22  \n",
            "   2    |  4300   |   0.097674   |     -      |     -     |   17.23  \n",
            "   2    |  4320   |   0.096278   |     -      |     -     |   17.22  \n",
            "   2    |  4340   |   0.083064   |     -      |     -     |   17.21  \n",
            "   2    |  4360   |   0.147637   |     -      |     -     |   17.23  \n",
            "   2    |  4380   |   0.120272   |     -      |     -     |   17.24  \n",
            "   2    |  4400   |   0.088911   |     -      |     -     |   17.23  \n",
            "   2    |  4420   |   0.138693   |     -      |     -     |   17.23  \n",
            "   2    |  4440   |   0.134326   |     -      |     -     |   17.21  \n",
            "   2    |  4460   |   0.121352   |     -      |     -     |   17.21  \n",
            "   2    |  4480   |   0.163713   |     -      |     -     |   17.26  \n",
            "   2    |  4500   |   0.058168   |     -      |     -     |   17.18  \n",
            "   2    |  4520   |   0.085916   |     -      |     -     |   17.18  \n",
            "   2    |  4540   |   0.060903   |     -      |     -     |   17.20  \n",
            "   2    |  4560   |   0.080436   |     -      |     -     |   17.24  \n",
            "   2    |  4580   |   0.120357   |     -      |     -     |   17.21  \n",
            "   2    |  4600   |   0.132200   |     -      |     -     |   17.19  \n",
            "   2    |  4620   |   0.176231   |     -      |     -     |   17.20  \n",
            "   2    |  4640   |   0.045035   |     -      |     -     |   17.17  \n",
            "   2    |  4660   |   0.138956   |     -      |     -     |   17.29  \n",
            "   2    |  4680   |   0.140839   |     -      |     -     |   17.24  \n",
            "   2    |  4700   |   0.070001   |     -      |     -     |   17.21  \n",
            "   2    |  4720   |   0.079584   |     -      |     -     |   17.22  \n",
            "   2    |  4740   |   0.206187   |     -      |     -     |   17.27  \n",
            "   2    |  4760   |   0.131246   |     -      |     -     |   17.27  \n",
            "   2    |  4780   |   0.078993   |     -      |     -     |   17.21  \n",
            "   2    |  4800   |   0.075004   |     -      |     -     |   17.29  \n",
            "   2    |  4820   |   0.098154   |     -      |     -     |   17.21  \n",
            "   2    |  4840   |   0.097368   |     -      |     -     |   17.19  \n",
            "   2    |  4860   |   0.112208   |     -      |     -     |   17.19  \n",
            "   2    |  4880   |   0.099741   |     -      |     -     |   17.26  \n",
            "   2    |  4900   |   0.083016   |     -      |     -     |   17.25  \n",
            "   2    |  4920   |   0.105216   |     -      |     -     |   17.28  \n",
            "   2    |  4940   |   0.142016   |     -      |     -     |   17.24  \n",
            "   2    |  4960   |   0.110285   |     -      |     -     |   17.22  \n",
            "   2    |  4980   |   0.118087   |     -      |     -     |   17.28  \n",
            "   2    |  4999   |   0.107604   |     -      |     -     |   16.38  \n",
            "----------------------------------------------------------------------\n",
            "   2    |    -    |   0.112181   |  0.178148  |   94.84   |  4663.37 \n",
            "----------------------------------------------------------------------\n",
            "\n",
            "\n",
            "Training complete!\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "TypeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-14-13b5522ef7f5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     36\u001b[0m   \u001b[0mset_seed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m42\u001b[0m\u001b[0;34m)\u001b[0m    \u001b[0;31m# Set seed for reproducibility\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m   \u001b[0mbert_classifier\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscheduler\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minitialize_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 38\u001b[0;31m   \u001b[0mval_loss1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_accuracy1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbert_classifier\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_dataloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_dataloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mevaluation\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     39\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m   \u001b[0mval_loss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_loss1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTypeError\u001b[0m: 'NoneType' object is not iterable"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vshXkNsVQy3Q"
      },
      "source": [
        "## 4. Random Sampling Test data <a class=\"anchor\" id=\"section_4\"></a>\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "id": "7OPQZZ-aXfuR",
        "outputId": "db2d6883-2670-4c96-a305-bfec41a7bf76"
      },
      "source": [
        "df_whole.loc[remain_index,:]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>country</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>gable tostee dumped by' emotional starved' gi...</td>\n",
              "      <td>AU</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>news bites from a melbourne icon popping up i...</td>\n",
              "      <td>AU</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>made with love kaum celebrates the best of ba...</td>\n",
              "      <td>AU</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>this kid is going viral for his perfect revie...</td>\n",
              "      <td>AU</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>meghan markle targeted in cruel topless photo...</td>\n",
              "      <td>AU</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>874531</th>\n",
              "      <td>african economies suffer effects of global ec...</td>\n",
              "      <td>TZ</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>874532</th>\n",
              "      <td>is china setting a debt trap on keny loans fr...</td>\n",
              "      <td>TZ</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>874533</th>\n",
              "      <td>citi bank launches volunteer africa 2019 prog...</td>\n",
              "      <td>TZ</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>874534</th>\n",
              "      <td>africa's property market thrives on domestic ...</td>\n",
              "      <td>TZ</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>874535</th>\n",
              "      <td>taxes force gamers to revert to unregistered ...</td>\n",
              "      <td>TZ</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>774535 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                     text country  label\n",
              "0        gable tostee dumped by' emotional starved' gi...      AU      0\n",
              "1        news bites from a melbourne icon popping up i...      AU      0\n",
              "2        made with love kaum celebrates the best of ba...      AU      0\n",
              "3        this kid is going viral for his perfect revie...      AU      0\n",
              "4        meghan markle targeted in cruel topless photo...      AU      0\n",
              "...                                                   ...     ...    ...\n",
              "874531   african economies suffer effects of global ec...      TZ      1\n",
              "874532   is china setting a debt trap on keny loans fr...      TZ      1\n",
              "874533   citi bank launches volunteer africa 2019 prog...      TZ      1\n",
              "874534   africa's property market thrives on domestic ...      TZ      1\n",
              "874535   taxes force gamers to revert to unregistered ...      TZ      1\n",
              "\n",
              "[774535 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 83
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2jpwr0zQXwKy",
        "outputId": "97846215-bb30-4665-87b2-da4ee8d056c4"
      },
      "source": [
        "remain_index = list(set(df_whole.index) - set(index))\n",
        "peace_index = random.sample(list(df_whole.loc[remain_index,:][df_whole['label'] == 0].index), 25000)\n",
        "nonpeaceful_index = random.sample(list(df_whole.loc[remain_index,:][df_whole['label'] == 1].index), 25000)\n",
        "\n",
        "ran_index = peace_index + nonpeaceful_index"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
            "  \n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:3: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
            "  This is separate from the ipykernel package so we can avoid doing imports until\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pGPdeJ-6DYJJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2b95c787-e501-43b6-a7ab-d0b655fab8dd"
      },
      "source": [
        "X_test = df_whole['text'][ran_index]\n",
        "y1_test = df_whole['label'][ran_index]\n",
        "X_test.index = np.arange(0,len(X_test))\n",
        "y1_test.index = np.arange(0,len(y1_test))\n",
        "\n",
        "test_inputs, test_masks = preprocessing_for_bert(X_test)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/transformers/tokenization_utils_base.py:2022: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8zN0C0iHDY3K"
      },
      "source": [
        "#Data Loader Class\n",
        "test_labels = torch.tensor(y1_test.values)\n",
        "test_data = TensorDataset(test_inputs, test_masks, test_labels)\n",
        "test_sampler = RandomSampler(test_data)\n",
        "test_dataloader = DataLoader(test_data, sampler = test_sampler, batch_size = batch_size)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B1xI5mM0Q4Gr"
      },
      "source": [
        "## 5. Separate Prediction model <a class=\"anchor\" id=\"section_5\"></a>\n",
        "\n",
        "Since BERT evaluates with random batches, we need to implement separate function for the prediction."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "brNFmZQ7_cMR"
      },
      "source": [
        "def bert_predict(model, test_dataloader):\n",
        "    \"\"\"Perform a forward pass on the trained BERT model to predict probabilities\n",
        "    on the test set.\n",
        "    \"\"\"\n",
        "    # Put the model into the evaluation mode. The dropout layers are disabled during\n",
        "    # the test time.\n",
        "    model.eval()\n",
        "    val_accuracy = []\n",
        "    all_pred = []\n",
        "    all_real = []\n",
        "    for batch in test_dataloader:\n",
        "        # Load batch to GPU\n",
        "        b_input_ids, b_attn_mask, b_labels = tuple(t.to(device) for t in batch)\n",
        "        all_real.append(b_labels)\n",
        "\n",
        "        # Compute logits\n",
        "        with torch.no_grad():\n",
        "            logits = model(b_input_ids, b_attn_mask)\n",
        "\n",
        "\n",
        "        # Get the predictions\n",
        "        preds = torch.argmax(logits, dim=1).flatten()\n",
        "        all_pred.append(preds)\n",
        "\n",
        "        # Calculate the accuracy rate\n",
        "        accuracy = (preds == b_labels).cpu().numpy().mean() * 100\n",
        "        val_accuracy.append(accuracy)\n",
        "\n",
        "    all_real = torch.cat(all_real)\n",
        "    all_pred = torch.cat(all_pred)\n",
        "    # Compute the average accuracy and loss over the validation set.\n",
        "    #val_loss = np.mean(val_loss)\n",
        "    val_accuracy = np.mean(val_accuracy)\n",
        "\n",
        "    return val_accuracy,all_real, all_pred"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xTTo7cZMGvx5"
      },
      "source": [
        "acc,all_real, all_pred = bert_predict(bert_classifier, test_dataloader)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CbkmnMCIFF2H"
      },
      "source": [
        "## 6. Evaluation Metrics <a class=\"anchor\" id=\"section_6\"></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s7kdi-_ENvN6"
      },
      "source": [
        "y_true = all_real.cpu().numpy()\n",
        "y_pred = all_pred.cpu().numpy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1bysy35wDs6j",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "89cfd54c-fd17-471d-8990-7e31c614d852"
      },
      "source": [
        "from sklearn.metrics import classification_report\n",
        "#0: peaceful, 1:non_peaceful\n",
        "\n",
        "print(classification_report(y_true, y_pred))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.95      0.95     25000\n",
            "           1       0.95      0.95      0.95     25000\n",
            "\n",
            "    accuracy                           0.95     50000\n",
            "   macro avg       0.95      0.95      0.95     50000\n",
            "weighted avg       0.95      0.95      0.95     50000\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 706
        },
        "id": "luW1wFiDErt-",
        "outputId": "ba98b328-2dad-4058-a3f3-0473773eead1"
      },
      "source": [
        "import itertools\n",
        "def plot_confusion_matrix(cm, classes,normalize=False):\n",
        "    plt.figure(figsize = (10,10))\n",
        "    plt.imshow(cm, interpolation='nearest', cmap=plt.cm.Blues)\n",
        "    plt.title('Confusion matrix')\n",
        "    plt.colorbar()\n",
        "    tick_marks = np.arange(len(classes))\n",
        "    plt.xticks(tick_marks, classes, rotation=90)\n",
        "    plt.yticks(tick_marks, classes)\n",
        "    if normalize:\n",
        "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
        "\n",
        "    thresh = cm.max() / 2.\n",
        "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
        "        plt.text(j, i, cm[i, j],\n",
        "                 horizontalalignment=\"center\",\n",
        "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
        "    plt.tight_layout()\n",
        "    plt.ylabel('Actual')\n",
        "    plt.xlabel('Predicted')\n",
        "\n",
        "dict_characters = {0: 'peaceful', 1: 'non-peaceful'}\n",
        "\n",
        "# y_pred = model.predict(test_data)\n",
        "# y_pred_classes = np.argmax(y_pred, axis=1)\n",
        "# y_true = np.argmax(test_labels, axis=1)\n",
        "\n",
        "confusion_mat = confusion_matrix(y_true, y_pred)\n",
        "plot_confusion_matrix(confusion_mat, classes = list(dict_characters.values()))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsEAAAKxCAYAAACyk3G1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzde/yt5Zg/8M+19+6oqHRASSSHGJpKNTlFdCCDGaPICBHj9BuMwRgVaZyPOU1GqWEq41RS0kSGTLQj5RAaQuk4peNIh/v3x/fZ27dmn3raa6/99LzfXuu113M/93rWtVavr33t63vd91OttQAAwJjMmXYAAACwokmCAQAYHUkwAACjIwkGAGB0JMEAAIyOJBgAgNGZN+0AAADoZ+5d79PaTf871Rja/152Umttt6kG0YMkGABgoNpN/5vVHvjMqcbw+7M+vP5UA+hJOwQAAKOjEgwAMFiVlJpmH741AABGRxIMAMDoaIcAABiqSlI17SgGSSUYAIDRkQQDADA62iEAAIbM7hC9+NYAABgdlWAAgCGzMK4XlWAAAEZHEgwAwOhohwAAGCy3Te7LtwYAwOioBAMADJmFcb2oBAMAMDqSYAAARkc7BADAUFUsjOvJtwYAwOhIggEAGB3tEAAAg1V2h+hJJRgAgNFRCQYAGDIL43rxrQEAMDqSYAAARkc7BADAkFkY14tKMAAAoyMJBgBgdLRDAAAMVtkdoiffGgAAo6MSDAAwVBUL43pSCQYAYHQkwQAAjI52CACAIbMwrhffGgAAoyMJBgBgdLRDAAAMln2C+/KtAQAwOirBAABDNsc+wX2oBAMAMDqSYAAARkc7BADAUFUsjOvJtwYAwOioBAMADFlZGNeHSjAAAKMjCQYAYHS0QwAADJY7xvXlWwMAYHQkwQAAjI52CACAIbM7RC8qwQAAjI5KMADAkFkY14tvDQCA0ZEEAwAwOtohAACGqsrCuJ5UggEAGB1JMAAAo6MdAgBgyOwO0YtvDQCA0VEJBgAYMgvjelEJBgBgdCTBAACMjnYIAIDBKgvjevKtAQAwOpJgAABGRzsEAMCQ2R2iF5VgAABGRyUYAGCoKhbG9eRbAwBgdCTBwGhU1RpV9aWquqqq/v0OXGfvqvrq8oxtWqrq0VX102nHAbCiSYKBlU5VPbuq5lfVtVV1UVWdWFWPWg6XfkaSjZLcvbX2V30v0lr7dGttl+UQz0RVVauq+y9pTmvtm621B66omIDlrdsneJqPgRpu5MCdUlW9Osn7k/xTZhLWTZN8JMlTl8Pl75PkZ621m5bDtQavqqwLAUZLEgysNKrqbknekuRlrbXPt9aua63d2Fr7Umvttd2c1arq/VX12+7x/qparTu3U1VdUFWvqapLuyry87tzb06yf5I9uwrzvlV1YFV9atb7b9ZVT+d1x8+rql9U1TVV9cuq2nvW+LdmvW7Hqjqja7M4o6p2nHXu1Ko6qKpO667z1apafzGff0H8fz8r/qdV1ZOq6mdVdUVV/cOs+dtV1X9V1e+6uR+qqlW7c//ZTftB93n3nHX911XVxUkOXzDWvWbz7j227o7vVVWXVdVOd+g/LDBZVdN9DJQkGFiZ/FmS1ZN8YQlz3phkhyRbJXl4ku2S/OOs8/dIcrckGyfZN8mHq2rd1toBmakuH9NaW6u19oklBVJVd0nywSS7t9bWTrJjkrMWMW+9JF/u5t49yXuTfLmq7j5r2rOTPD/JhklWTfJ3S3jre2TmO9g4M0n7x5M8J8k2SR6d5E1Vdd9u7s1JXpVk/cx8dzsneWmStNYe0815ePd5j5l1/fUyUxXfb/Ybt9b+O8nrknyqqtZMcniSI1prpy4hXoBBkgQDK5O7J7l8Ke0Keyd5S2vt0tbaZUnenOSvZ52/sTt/Y2vthCTXJunb83pLkodW1RqttYtaaz9axJwnJ/l5a+1fW2s3tdaOSnJukqfMmnN4a+1nrbX/TfKZzCTwi3NjkoNbazcmOTozCe4HWmvXdO//48wk/2mtndlaO7173/OT/HOSxy7DZzqgtXZDF8+ttNY+nuS8JN9Jcs/M/KMD4E5HEgysTP4nyfpL6VW9V5JfzTr+VTe28Bq3SaKvT7LW7Q2ktXZdkj2TvCTJRVX15ap60DLEsyCmjWcdX3w74vmf1trN3fMFSeols87/74LXV9UDqur4qrq4qq7OTKV7ka0Ws1zWWvv9UuZ8PMlDkxzSWrthKXOBabMwrpfhRg7cGf1XkhuSPG0Jc36bmV/lL7BpN9bHdUnWnHV8j9knW2sntdaemJmK6LmZSQ6XFs+CmC7sGdPt8dHMxLVFa+2uSf4hM1vnL0lb0smqWiszCxM/keTArt0D4E5HEgysNFprV2WmD/bD3YKwNatqlaravare2U07Ksk/VtUG3QKz/ZN8anHXXIqzkjymqjbtFuW9YcGJqtqoqp7a9QbfkJm2ilsWcY0Tkjyg29ZtXlXtmWTLJMf3jOn2WDvJ1Umu7arUf3Ob85ckud/tvOYHksxvrb0wM73OH7vDUQKshCTBwEqltfaeJK/OzGK3y5L8JsnLk3yxm/LWJPOTnJ3knCTf68b6vNfJSY7prnVmbp24zuni+G2SKzLTa3vbJDOttf9JskeS12SmnePvk+zRWru8T0y3099lZtHdNZmpUh9zm/MHJjmi2z3imUu7WFU9Nclu+ePnfHWSrRfsigGspOwO0Uu1tsTfjAEAsJKas8592mo7TXf96u+PffGZrbVtpxpEDzZKBwAYqqpBL06bJt8aAACjIwkGAGB0tEMAAAzZgBenTZNKMAAAo6MSPCU1b41Wq6497TCACfrTB2867RCACfvVr87P5ZdfrhQ7QJLgKalV185qD1zqtp3AgJ32nQ9NOwRgwh65/fR3BivtEL1ohwAAYHRUggEABqqiEtyXSjAAAKMjCQYAYHS0QwAADFV1D243lWAAAEZHEgwAwOhohwAAGKyyO0RPKsEAAIyOSjAAwICpBPejEgwAwOhIggEAGB3tEAAAA6Ydoh+VYAAARkclGABgwFSC+1EJBgBgdCTBAACMjnYIAIChqu7B7aYSDADA6EiCAQAYHUkwAMBAVSpV030sNcaqe1fV16vqx1X1o6r6f934elV1clX9vPtz3W68quqDVXVeVZ1dVVvPutY+3fyfV9U+s8a3qapzutd8sJYhMEkwAACTdFOS17TWtkyyQ5KXVdWWSV6f5JTW2hZJTumOk2T3JFt0j/2SfDSZSZqTHJBk+yTbJTlgQeLczXnRrNfttrSgJMEAAAO2sleCW2sXtda+1z2/JslPkmyc5KlJjuimHZHkad3zpyY5ss04Pck6VXXPJLsmObm1dkVr7cokJyfZrTt319ba6a21luTIWddaLEkwAAArRFVtluRPk3wnyUattYu6Uxcn2ah7vnGS38x62QXd2JLGL1jE+BLZIg0AgDti/aqaP+v40NbaobedVFVrJflckr9trV09u4rcWmtV1SYf6h9JggEABmwluG3y5a21bZc0oapWyUwC/OnW2ue74Uuq6p6ttYu6loZLu/ELk9x71ss36cYuTLLTbcZP7cY3WcT8JdIOAQDAxHQ7NXwiyU9aa++ddeq4JAt2eNgnybGzxp/b7RKxQ5KruraJk5LsUlXrdgvidklyUnfu6qraoXuv58661mKpBAMAMEmPTPLXSc6pqrO6sX9I8vYkn6mqfZP8Kskzu3MnJHlSkvOSXJ/k+UnSWruiqg5KckY37y2ttSu65y9N8skkayQ5sXsskSQYAGDAVoJ2iCVqrX0ri7+5886LmN+SvGwx1zosyWGLGJ+f5KG3Jy7tEAAAjI5KMADAUFUWX2NliVSCAQAYHUkwAACjox0CAGDAVvaFcSsrlWAAAEZHEgwAwOhohwAAGKhKaYfoSSUYAIDRUQkGABgwleB+VIIBABgdSTAAAKOjHQIAYMh0Q/SiEgwAwOioBAMADFVZGNeXSjAAAKMjCQYAYHS0QwAADJh2iH5UggEAGB1JMAAAo6MdAgBgwLRD9KMSDADA6KgEAwAMVKVUgntSCQYAYHQkwQAAjI52CACAIdMN0YtKMAAAoyMJBgBgdLRDAAAMVdknuC+VYAAARkclGABgwFSC+1EJBgBgdCTBAACMjnYIAIAB0w7Rj0owAACjIwkGAGB0tEMAAAyZboheVIIBABgdlWAAgAGzMK4flWAAAEZHEgwAwOhohwAAGKiq0g7Rk0owAACjoxIMADBgKsH9qAQDADA6kmAAAEZHOwQAwIBph+hHJRgAgNGRBAMAMDraIQAAhkw3RC8qwQAAjI5KMADAgFkY149KMAAAoyMJBgBgdLRDAAAMVWmH6EslGACA0ZEEAwAwOtohAAAGqpLohuhHJRgAgNFRCQYAGKyyMK4nlWAAAEZHEgwAwOhohwAAGDDdEP2oBAMAMDqSYAAARkc7BADAgNkdoh+VYAAARkclGABgqMrCuL5UggEAGB1JMAAAo6MdAgBgoCrJnDn6IfpQCQYAYHRUggEABszCuH5UggEAGB1JMAAAo6MdAgBgwNwxrh+VYAAARkcSDADA6GiHAAAYKrdN7k0lGACA0VEJBgAYqIqFcX2pBAMAMDqSYAAARkc7BADAYJV2iJ5UggEAGB1JMAAAo6MdAm6HTTZaJ/9y0HOz4d3XTmvJYZ87LR8+6tTs/9InZ4/HPiy3tJbLrrgm+x3wqVx02VV51XN3zp5PekSSZN7cOXnQfe+Rez/+9bny6uvzxB0fnHe/9hmZO2dOPvnFb+fdh5+cJDn0zc/Jo7e5f6669vdJkv32/9ec/bMLp/aZYexe/MIX5MQTjs8GG26YM8/6YZLkDa97bU748pey6iqr5r6bb55D/+XwrLPOOvnDH/6Ql//Ni/O9M+dnzpw5eff7PpDHPHanJMkuO++Uiy++KGusvkaS5EsnfjUbbrjhtD4WdyK6IfqRBMPtcNPNt+T17/18zjr3gqy15mr59r+9Lqd859y874hT8paPfDlJ8tJnPTZv2G/3vPLgo/O+I0/J+448JUnypMc8NK/Y+3G58urrM2dO5f2vf2ae/DcfyoWX/C7f+vRrc/w3zsm5v7g4SfIP7/9ivvAfZ03tcwJ/9Nf7PC8veenL88IXPHfh2M5PeGIOOvhtmTdvXt74htflXe94Ww5+2zty2L98PEky/6xzcumll+Zpe+yeb51+RubMmfnF6+FHfDrbbLvtVD4HcGvaIeB2uPjyq3PWuRckSa69/oac+8uLc68N1sk11/1+4Zw111gtrbX/89pn7rZtPvOVM5Mkj3joZvnv31ye8y/8n9x4083595O+lz12etiK+RDA7fKoRz8m66233q3GnvDEXTJv3kwdabvtd8iFF8z8/8K5P/lxdnrc45MkG264Ye62zjo5c/78FRswo1NVU30MlSQYetr0nutlqwdukjN+eH6S5MCXPSU/P/Gg7LX7tjnoo1++1dw1Vl8lT9zxwfniKTPV3XtteLdccMmVC89feMmV2XiDuy08PvBlT8l3j3lD3vmav8iqq/iFDazMjvzkYdl1t92TJH/ysIfn+OOPy0033ZTzf/nLfP97Z+aCC36zcO6LX/j8bL/NVnnbwQct8h/LwIojCV5GVfVXVfWTqvr6UuadX1Xrr6i4mI67rLFqjnr3C/Pad39uYRX4wA9/KVvs/qYcfeL8vGTPx9xq/pMf8yf5r7N+kSuvvn6p197/kOPy8KcflEc9511Z9253yWue/4SJfAbgjnvH2w7O3Hnzstez906S7PP8F2TjjTfJI7ffNq99zd9mhz/bMXPnzk2SHH7kpzP/rHPyH6d+M6d965v5t0/96zRDh9GTBC+7fZO8qLX2uGkHwnTNmzcnR737RTnmxPk59ms/+D/njznhjDxt561uNfZXu26Tf+9aIZLkt5delU02Wnfh8cYbrZsLL7sqyUzLRZL84cabcuSxp2fbh2w2gU8B3FH/esQnc8KXj88nj/z0wl8Jz5s3L+96z/vynTPPyr9//tj87ne/yxZbPCBJsvHGGydJ1l577ey517NzxhnfnVrs3InUzMK4aT6GanBJcFVtVlXnVtWnu8rsZ6tqzarapqq+UVVnVtVJVXXPbv6LquqMqvpBVX2uqtbsxjeqqi904z+oqh278edU1Xer6qyq+ueqmltV+yd5VJJPVNW7qup5VfWhWTEdX1U7TeHrYAo+dsDe+ekvL84HP/W1hWObb7rBwud77PSw/Oz8SxYe33Wt1fOobe6fL5169sKx+T/6Ve6/6Qa5z73unlXmzc1f7bp1vtydv8f6d104788f97D8+L9/O8mPA/Tw1ZO+kve+55357BeOy5prrrlw/Prrr891112XJDnlP07OvHnz8uAtt8xNN92Uyy+/PEly44035oQTjs9DHvLQqcQOzBhqs+EDk+zbWjutqg5L8rIkT0/y1NbaZVW1Z5KDk7wgyedbax9Pkqp6a2Yquock+WCSb7TWnl5Vc5OsVVUPTrJnkke21m6sqo8k2bu19paqenySv2utza+q563gz8tKYset7pe999g+5/zswpx+9OuTJAd86Lg872k7Zov7bJhbbmn59UVX5JUHH73wNX/+uIfnlNPPzfW//8PCsZtvviWvesdn8qWPvCxz51SOOPb0/KTbGeLwg/fJ+uuunark7J9ekFfMuhaw4j33Oc/KN79xai6//PJsvtkmedP+b8673vm23HDDDdljtycmmVkcd8hHPpbLLr00T3nyrpkzZ07uda+N84lPzrQ83HDDDfnzJ+2aG2+8MTffcnMe9/gn5AUvfNE0PxZ3EpUMenHaNNXQGvOrarMk/9la27Q7fnySf0iyXZJfdNPmJrmotbZLVT02yVuTrJNkrSQntdZeUlWXJdmktXbDrGu/vLvWpd3QGkmOaq0dWFWn5tZJ8LattZd3rzs+ybtba6dW1fnducsXEft+SfZLkqyy1jarP2Sf5fOlACulK8/40NInAYP2yO23zZlnzp9aFnqXjR/YHvSSj03r7ZMk39v/8We21ga3999QK8G3zdyvSfKj1tqfLWLuJ5M8rbX2gy553WkJ160kR7TW3rCU978pt24lWX0p85MkrbVDkxyaJHPW3HBY//oAALgTGVxPcGfTqlqQ8D47yelJNlgwVlWrVNVDuvNrJ7moqlZJsvesa5yS5G+6+XOr6m7d2DOqasNufL2qus8i3v/8JFtV1ZyqundmqtAAACuchXH9DDUJ/mmSl1XVT5Ksm5ke32ckeUdV/SDJWUl27Oa+Kcl3kpyW5NxZ1/h/SR5XVeckOTPJlq21Hyf5xyRfraqzk5yc5J6LeP/TkvwyyY8z01v8veX78QAAmKShtkPc1Fp7zm3GzkrymNtObK19NMlHFzF+SZKnLmL8mCTHLGJ8p1nPW25dVZ49b7Mlhw4AwLQNNQkGACB2h+hrcElwa+38JDZXBACgt8ElwQAA/JFCcD9DXRgHAAC9SYIBABgd7RAAAENVFsb1pRIMAMDoSIIBABgd7RAAAANVsTtEXyrBAACMjkowAMBglYVxPakEAwAwOpJgAABGRzsEAMCA6YboRyUYAIDRkQQDADA62iEAAAbM7hD9qAQDADA6KsEAAENVFsb1pRIMAMDoSIIBABgd7RAAAANVsTCuL5VgAABGRyUYAGDAVIL7UQkGAGB0JMEAAIyOdggAgAHTDdGPSjAAAKMjCQYAYHS0QwAADJjdIfpRCQYAYHRUggEAhqosjOtLJRgAgNGRBAMAMDraIQAABqpSFsb1pBIMAMDoSIIBABgd7RAAAAOmG6IflWAAAEZHJRgAYMDmKAX3ohIMAMDoSIIBABgdSTAAwIBVTfexbDHWYVV1aVX9cNbYgVV1YVWd1T2eNOvcG6rqvKr6aVXtOmt8t27svKp6/azx+1bVd7rxY6pq1aXFJAkGAGDSPplkt0WMv6+1tlX3OCFJqmrLJHsleUj3mo9U1dyqmpvkw0l2T7Jlkmd1c5PkHd217p/kyiT7Li0gSTAAABPVWvvPJFcs4/SnJjm6tXZDa+2XSc5Lsl33OK+19ovW2h+SHJ3kqTVzy7zHJ/ls9/ojkjxtaW8iCQYAGKiZloSa6uMOenlVnd21S6zbjW2c5Dez5lzQjS1u/O5Jftdau+k240skCQYA4I5Yv6rmz3rst4yv+2iSzZNsleSiJO+ZWISLYJ9gAIABmzP9bYIvb61te3tf1Fq7ZMHzqvp4kuO7wwuT3HvW1E26sSxm/H+SrFNV87pq8Oz5i6USDADACldV95x1+PQkC3aOOC7JXlW1WlXdN8kWSb6b5IwkW3Q7QayamcVzx7XWWpKvJ3lG9/p9khy7tPdXCQYAYKKq6qgkO2WmdeKCJAck2amqtkrSkpyf5MVJ0lr7UVV9JsmPk9yU5GWttZu767w8yUlJ5iY5rLX2o+4tXpfk6Kp6a5LvJ/nE0mKSBAMADNhyWJw2ca21Zy1ieLGJamvt4CQHL2L8hCQnLGL8F5nZPWKZaYcAAGB0VIIBAAZsAIXglZJKMAAAoyMJBgBgdLRDAAAMVCWp6IfoQyUYAIDRkQQDADA62iEAAAZsJbht8iCpBAMAMDoqwQAAQ1U1iDvGrYxUggEAGB1JMAAAo6MdAgBgwHRD9KMSDADA6EiCAQAYHe0QAAADVUnm6IfoRSUYAIDRUQkGABgwheB+VIIBABgdSTAAAKOjHQIAYMDcNrkflWAAAEZHEgwAwOhohwAAGKgqu0P0pRIMAMDoqAQDAAyYO8b1oxIMAMDoSIIBABgd7RAAAAOmGaIflWAAAEZHJRgAYMDcMa4flWAAAEZHEgwAwOhohwAAGKhKMkc3RC8qwQAAjI4kGACA0dEOAQAwVFV2h+hJJRgAgNFRCQYAGDCF4H5UggEAGB1JMAAAo6MdAgBgwCyM60clGACA0ZEEAwAwOtohAAAGym2T+1MJBgBgdFSCAQAGzMK4flSCAQAYHUkwAACjox0CAGDANEP0oxIMAMDoSIIBABgd7RAAAANVlcyxO0QvKsEAAIyOSjAAwIApBPejEgwAwOgsthJcVYckaYs731p75UQiAgCACVtSO8T8FRYFAAC9uG1yP4tNgltrR6zIQAAAYEVZ6sK4qtogyeuSbJlk9QXjrbXHTzAuAACWgUJwP8uyMO7TSX6S5L5J3pzk/CRnTDAmAACYqGVJgu/eWvtEkhtba99orb0giSowAACDtSz7BN/Y/XlRVT05yW+TrDe5kAAAWBaVcse4npYlCX5rVd0tyWuSHJLkrkleNdGoAABggpaaBLfWju+eXpXkcZMNBwAAJm9Zdoc4PIu4aUbXGwwAwLSU3SH6WpZ2iONnPV89ydMz0xcMAACDtCztEJ+bfVxVRyX51sQiAgBgmbljXD/LskXabW2RZMPlHQgAAKwoy9ITfE1u3RN8cWbuIMcd8KcP3jSnfedD0w4DmKB1H/HyaYcATNgNP/31tEOgp2Vph1h7RQQCAMDt1+fX+izD91ZVpyzLGAAADMViK8FVtXqSNZOsX1XrJlnQdX3XJBuvgNgAAGAiltQO8eIkf5vkXknOzB+T4KuTaGYFAJiyit0h+lpsEtxa+0CSD1TVK1prh6zAmAAAYKKWpZf6lqpaZ8FBVa1bVS+dYEwAACyjOTXdx1AtSxL8otba7xYctNauTPKiyYUEAACTtSxJ8Nya1WxSVXOTrDq5kAAAYLKWuk9wkq8kOaaq/rk7fnGSEycXEgAAy2rILQnTtCxJ8OuS7JfkJd3x2UnuMbGIAABgwpbaDtFauyXJd5Kcn2S7JI9P8pPJhgUAAJOzpJtlPCDJs7rH5UmOSZLW2uNWTGgAACxJlX2C+1pSO8S5Sb6ZZI/W2nlJUlWvWiFRAQDABC0pCf6LJHsl+XpVfSXJ0fnjXeMAAFgJWBjXz2J7gltrX2yt7ZXkQUm+nplbKG9YVR+tql1WVIAAALC8LcvCuOtaa//WWntKkk2SfD8zO0YAAMAgLcsWaQt1d4s7tHsAADBl1sX1syx3jAMAgDuV21UJBgBg5VFJ5igF96ISDADA6EiCAQAYHe0QAAADpqLZj+8NAIDRkQQDADA62iEAAAbM5hD9qAQDADA6KsEAAANVVfYJ7kklGACA0ZEEAwAwOtohAAAGTDdEPyrBAACMjiQYAIDR0Q4BADBgc7RD9KISDADA6KgEAwAMVCX2Ce5JJRgAgNGRBAMAMDraIQAABkw3RD8qwQAAjI4kGACA0dEOAQAwVGWf4L5UggEAGB2VYACAAasoBfehEgwAwOhIggEAGB3tEAAAAzVz2+RpRzFMKsEAAIyOSjAAwICpBPejEgwAwOhIggEAGB3tEAAAA1alH6IPlWAAAEZHEgwAwOhohwAAGCj7BPenEgwAwOioBAMADFUl1sX1oxIMAMDoSIIBABgd7RAAAAM2Rz9ELyrBAACMjiQYAIDR0Q4BADBQ9gnuTyUYAIDRUQkGABgw6+L6UQkGAGB0JMEAAIyOdggAgMGqzIl+iD5UggEAGB1JMAAAo6MdAgBgoCp2h+hLJRgAgImqqsOq6tKq+uGssfWq6uSq+nn357rdeFXVB6vqvKo6u6q2nvWafbr5P6+qfWaNb1NV53Sv+WDV0v9pIAkGABiqmrlj3DQfy+iTSXa7zdjrk5zSWtsiySndcZLsnmSL7rFfko8mM0lzkgOSbJ9kuyQHLEicuzkvmvW6277X/yEJBgBgolpr/5nkitsMPzXJEd3zI5I8bdb4kW3G6UnWqap7Jtk1ycmttStaa1cmOTnJbt25u7bWTm+ttSRHzrrWYkmCAQCYho1aaxd1zy9OslH3fOMkv5k174JubEnjFyxifIksjAMAGLA5018Zt35VzZ91fGhr7dDbc4HWWquqtpzjWiJJMAAAd8TlrbVte7zukqq6Z2vtoq6l4dJu/MIk9541b5Nu7MIkO91m/NRufJNFzF8i7RAAAAO1YIu0aT7ugOOSLNjhYZ8kx84af263S8QOSa7q2iZOSrJLVa3bLYjbJclJ3bmrq2qHbleI58661mKpBAMAMFFVdVRmqrjrV9UFmdnl4e1JPlNV+yb5VZJndtNPSPKkJOcluT7J85OktXZFVR2U5LXboekAABnGSURBVIxu3ltaawsW2700MztQrJHkxO6xRJJgAAAmqrX2rMWc2nkRc1uSly3mOoclOWwR4/OTPPT2xCQJBgAYsJVgYdwg6QkGAGB0JMEAAIyOdggAgAHTDdGPSjAAAKOjEgwAMFAVFc2+fG8AAIyOJBgAgNHRDgEAMFSVlJVxvagEAwAwOpJgAABGRzsEAMCAaYboRyUYAIDRUQkGABioSjLHwrheVIIBABgdSTAAAKOjHQIAYMA0Q/SjEgwAwOioBAMADJh1cf2oBAMAMDqSYAAARkc7BADAYFVKP0QvKsEAAIyOJBgAgNHRDgEAMFAVFc2+fG8AAIyOSjAAwIBZGNePSjAAAKMjCQYAYHS0QwAADJhmiH5UggEAGB1JMAAAo6MdAgBgqMruEH2pBAMAMDoqwQAAA+WOcf353gAAGB1JMAAAo6MdAgBgwCyM60cSDD29+IUvyIknHJ8NNtwwZ571wyTJG1732pzw5S9l1VVWzX033zyH/svhWWeddRa+5te//nW2ftiWeeP+B+ZVr/67xV4HmJ5NNlon/3LQc7Ph3ddOa8lhnzstHz7q1Oz/0idnj8c+LLe0lsuuuCb7HfCpXHTZVXnVc3fOnk96RJJk3tw5edB975F7P/71ufLq63Pul9+ca667ITffcktuuvmWPGrvdyZJ/uQBG+eQN+6Vu6yxWn712//J8994RK657vfT/NgwOtohoKe/3ud5Ofb4r9xqbOcnPDFnnvXDnPH9s7PFFg/Iu97xtludf91rX51ddtt9qdcBpuemm2/J69/7+Wz9lwfnsc99d16852PyoPvdI+874pRst+fbssNeb8+J3/xh3rDfzM/y+448JTvs9fbssNfbs/8hx+WbZ/48V159/cLr7bbfB7LDXm9fmAAnyUf3f3b+8YPH5hHP/Kcc9/Uf5FX77LzCPyeMnSQYenrUox+T9dZb71ZjT3jiLpk3b+YXLNttv0MuvOCCheeOO/aL2Wyz+2bLLR+y1OsA03Px5VfnrHNnfnavvf6GnPvLi3OvDda5VaV2zTVWS2vt/7z2mbttm8985cylvsf9N90w3zrzvCTJ104/N0/beavlFD1jVFN+DJUkGCbkyE8ell27qu+1116b97zrHXnjmw6YclTA7bHpPdfLVg/cJGf88PwkyYEve0p+fuJB2Wv3bXPQR798q7lrrL5Knrjjg/PFU85aONZay5c+8vKc9um/zwv+4pELx3/yi4vylJ0eliT5iydunU02WnfyHwa4FUnwMqqqDarqO1X1/ap69BLmHVhVf7ciY2Pl8463HZy58+Zlr2fvnSR561sOzCv+36uy1lprTTkyYFndZY1Vc9S7X5jXvvtzC6vAB374S9li9zfl6BPn5yV7PuZW85/8mD/Jf531i1u1Quz8/Pdlx2e/I097+Ufy4j0fnUduvXmS5MUHfjr7PfPROe3Tf5+11lwtf7jx5hX3wbjTqZruY6gsjFt2Oyc5p7X2wmkHwsrtX4/4ZE748vE58aunLFyxe8Z3v5MvfP6zeeMb/j5X/e53mTNnTlZfbfX8zctePuVogUWZN29Ojnr3i3LMifNz7Nd+8H/OH3PCGfnCIX+Tt37shIVjf7XrNvn327RC/Payq5Ikl115bY772tl5xEM2y2nf++/87PxL8pSXfjjJTGvE7o++dZsUMHkTqwRX1WZV9ZOq+nhV/aiqvlpVa1TVVlV1elWdXVVfqKp1u/mnVtU7quq7VfWzxVVbu3kfqKqzquqHVbVdN36Xqjqse/33q+qps+L4ZlV9r3vsOOtar6uqc6rqB1X19m5s86r6SlWd2b3uQVW1VZJ3Jnlq975rVNW1s67zjKr65KS+S4bjqyd9Je99zzvz2S8clzXXXHPh+CmnfjM/Pe/8/PS88/PyV/5tXvv6f5AAw0rsYwfsnZ/+8uJ88FNfWzi2+aYbLHy+x04Py8/Ov2Th8V3XWj2P2ub++dKpZy8cW3P1VbPWmqstfP6EP3tQfvTfv02SbLDuzG+Fqiqvf9Gu+fhnvzXRzwP8X5OuBG+R5FmttRdV1WeS/GWSv0/yitbaN6rqLUkOSPK3C+JprW1XVU/qxp+wmOuu2Vrbqqoek+SwJA9N8sYkX2utvaCq1kny3ar6jySXJnlia+33VbVFkqOSbFtVuyd5apLtW2vXV9WClUmHJnlJa+3nVbV9ko+01h5fVfsn2ba19vLEnnwkz33Os/LNb5yayy+/PJtvtknetP+b8653vi033HBD9tjtiUlmFscd8pGP3e7rPO8F+66IjwAswo5b3S9777F9zvnZhTn96NcnSQ740HF53tN2zBb32TC33NLy64uuyCsPPnrha/78cQ/PKaefm+t//4eFYxvefe0c894XJUnmzZ2bY06cn5O//ZMkMwvoXty1Uxz7tbNy5LGnr6iPx53MzG2T5SR91KJWty6XC1dtluTk1toW3fHrkqyeZN/W2qbd2OZJ/r21tnVVnZrkja2106pqoySntdbuv4jrnprkLa21r3XHv07ysCT/0V3/pm7qekl2TfLbJB9KslWSm5M8oLW2ZlW9J8m5rbWPz7r2WkkuS/LTWW+5WmvtwVX1vNw6Cb62tbZW9/wZSfZorT2vqg5Mcm1r7d2LiH2/JPslyb033XSbn/33r5b16wQGaN1HqPbDnd0NP/1Mbrn+0qlloVs85OHtvUd/dVpvnyT584fd48zW2rZTDaKHSVeCb5j1/OYk6yxu4m3m35wutqo6PMmfJvlta+1J3fnbZu4tM/8Y+svW2uwENl1SekmSh2em/WNJu5HPSfK71tqy7FUzO4bVl2F+WmuHZqbSnG222XYy//oAAEbFL6f7WdG7Q1yV5MpZ/b5/neQbS3pBa+35rbWtZiXASbJnklTVo5Jc1Vq7KslJSV5RXZ9CVf1pN/duSS5qrd3Svd/cbvzkJM+vqjW7+eu11q5O8suq+qturKrq4YsJ7ZKqenBVzUny9GX9AgAAmL5pbJG2T5J3VdXZmWlReEuPa/y+qr6f5GNJFjRPHpRklSRnV9WPuuMk+UiSfarqB0kelOS6JGmtfSXJcUnmV9VZSRZsa7Z3kn27+T/KTN/worw+yfFJvp3koh6fAQCAKZlYT/CkdD3Bf9damz/tWO6IbbbZtp32nUF/BGAp9ATDnd/0e4K3au8/Zro9wXv8yUaD7Al2swwAAEZncDfLaK3tNO0YAAAYtsElwQAA/JHdIfrRDgEAwOioBAMADJQ7xvWnEgwAwOhIggEAGB3tEAAAQ1UWxvWlEgwAwOhIggEAGB3tEAAAA6Ydoh+VYAAARkclGABgwMo+wb2oBAMAMDqSYAAARkc7BADAQFWSObohelEJBgBgdCTBAACMjnYIAIABsztEPyrBAACMjkowAMCAuWNcPyrBAACMjiQYAIDR0Q4BADBgFsb1oxIMAMDoqAQDAAyUO8b1pxIMAMDoSIIBABgd7RAAAINVFsb1pBIMAMDoSIIBABgd7RAAAENVbpvcl0owAACjoxIMADBgCsH9qAQDADA6kmAAAEZHOwQAwEDN3DZZQ0QfKsEAAIyOJBgAgNHRDgEAMGCaIfpRCQYAYHRUggEAhkwpuBeVYAAARkcSDADA6GiHAAAYsNIP0YtKMAAAoyMJBgBgdLRDAAAMmLsm96MSDADA6KgEAwAMmEJwPyrBAACMjiQYAIDR0Q4BADBk+iF6UQkGAGB0VIIBAAaq4o5xfakEAwAwOpJgAABGRzsEAMBQlTvG9aUSDADA6EiCAQAYHe0QAAADphuiH5VgAABGRyUYAGDIlIJ7UQkGAGB0JMEAAIyOdggAgMEqt03uSSUYAIDRkQQDADA62iEAAAbMbZP7UQkGAGB0VIIBAAaqYpvgvlSCAQAYHUkwAACjox0CAGDI9EP0ohIMAMDoSIIBABgd7RAAAAPmtsn9qAQDADA6KsEAAAPmjnH9qAQDADA6kmAAAEZHOwQAwIDphuhHJRgAgNFRCQYAGKqKUnBPKsEAAIyOJBgAgNHRDgEAMGDuGNePSjAAAKMjCQYAYHS0QwAADFTFbZP7UgkGAGCiqur8qjqnqs6qqvnd2HpVdXJV/bz7c91uvKrqg1V1XlWdXVVbz7rOPt38n1fVPnckJkkwAMCA1ZQft8PjWmtbtda27Y5fn+SU1toWSU7pjpNk9yRbdI/9knw0mUmakxyQZPsk2yU5YEHi3IckGACAaXhqkiO650ckedqs8SPbjNOTrFNV90yya5KTW2tXtNauTHJykt36vrkkGACASWtJvlpVZ1bVft3YRq21i7rnFyfZqHu+cZLfzHrtBd3Y4sZ7sTAOAGDIpr8wbv0Ffb6dQ1trh95mzqNaaxdW1YZJTq6qc2efbK21qmoTj3QWSTAAAHfE5bP6fBeptXZh9+elVfWFzPT0XlJV92ytXdS1O1zaTb8wyb1nvXyTbuzCJDvdZvzUvkFrhwAAYGKq6i5VtfaC50l2SfLDJMclWbDDwz5Jju2eH5fkud0uETskuaprmzgpyS5VtW63IG6XbqwXlWAAgAEbwG2TN0ryhZrZ0Hhekn9rrX2lqs5I8pmq2jfJr5I8s5t/QpInJTkvyfVJnp8krbUrquqgJGd0897SWruib1CSYAAAJqa19oskD1/E+P8k2XkR4y3JyxZzrcOSHLY84pIEAwAMmDvG9aMnGACA0ZEEAwAwOtohAAAGTDdEPyrBAACMjiQYAIDR0Q4BADBk+iF6UQkGAGB0VIIBAAaqMog7xq2UVIIBABgdSTAAAKOjHQIAYKjKbZP7UgkGAGB0VIIBAAZMIbgflWAAAEZHEgwAwOhohwAAGDL9EL2oBAMAMDqSYAAARkc7BADAYJXbJvekEgwAwOioBE/J97535uVrrFK/mnYcrFDrJ7l82kEAE+XnfHzuM+0A3DGuH0nwlLTWNph2DKxYVTW/tbbttOMAJsfPOQyHdggAAEZHJRgAYKAqtgnuSyUYVpxDpx0AMHF+zmEgJMGwgrTW/OUId3J+zmE4tEMAAAyZfoheVIIBABgdlWAAgAFzx7h+JMEwAVW13pLOt9auWFGxAJNRVa9e0vnW2ntXVCzA7ScJhsk4M0nLoju1WpL7rdhwgAlYe9oBAP1JgmECWmv3nXYMwGS11t487RggcdvkviTBMEFV9ZhFjbfW/nNFxwJMRlUdnpnf8NxKa+0FUwgHWEaSYJis1856vnqS7TLTKvH46YQDTMDxs56vnuTpSX47pViAZSQJhglqrT1l9nFV3TvJ+6cUDjABrbXPzT6uqqOSfGtK4TBCuiH6sU8wrFgXJHnwtIMAJmqLJBtOOwhgyVSCYYKq6pD8sVdwTpKtknxvehEBy1tVXZNb9wRfnOR1UwqHsSkL4/qSBMNkzZ/1/KYkR7XWTptWMMDyU1WP7H6eN2it/X7a8QC3jyQYJqCqTmmt7Zxky9aaihDcOX0wyTZJvp1k6ynHAtxOkmCYjHtW1Y5J/ryqjs5t1i201rREwPDdWFWHJtmkqj5425OttVdOISZGST9EH5JgmIz9k7wpySZJbnvr1BZbpMGdwR5JnpBk18xsfQgMiCQYJqC19tkkn62qN7XWDpp2PMDy11q7PMnRVfWT1toPph0P41SxMK4vW6TBZB1cVc+pqv2TpKo2rartph0UsFz9b1WdUlU/TJKqelhV/eO0gwKWTBIMk/XhJH+W5Fnd8TXdGHDn8fEkb0hyY5K01s5OstdUIwKWSjsETNb2rbWtq+r7SdJau7KqVp12UMBytWZr7bt1699J3zStYBgf3RD9qATDZN1YVXPTbaRfVRskuWW6IQHL2eVVtXn++HP+jCQXTTckYGlUgmGyPpjkC0k2rKqDkzwjiV5BuHN5WZJDkzyoqi5M8sskz5luSMDSSIJhglprn66qM5PsnJnfWD2ttfaTKYcFLEettV8keUJV3SXJnNbaNdOOiXGxO0Q/2iFggqpqhyQXttY+3Fr7UJILq2r7accFLD9V9U9VtU5r7brW2jVVtW5VvXXacQFLJgmGyfpokmtnHV/bjQF3Hru31n634KC1dmWSJ00xHkampvy/oZIEw2RVa60tOGit3RJtSHBnM7eqVltwUFVrJFltCfOBlYAkGCbrF1X1yqpapXv8vyS/mHZQwHL16SSnVNW+VbVvkpOTHDHlmIClUJGCyXpJZnaI+MfMbJ90SpL9phoRsFy11t5RVWdnZgFskhzUWjtpmjExMsPtSJgqSTBMUGvt0rhzFNzptdZOTHLitOMAlp0kGCaoqlZPsm+ShyRZfcF4a+0FUwsKWK66XWAOSfLgJKsmmZvkutbaXacaGLBEeoJhsv41yT2S7JrkG0k2SWIPUbhz+VCSZyX5eZI1krwwyYenGhGjUlN+DJUkGCbr/q21N2WmKnREkicnsU8w3Mm01s5LMre1dnNr7fAku007JmDJtEPAZN3Y/fm7qnpokouTbDjFeIDl7/qqWjXJWVX1ziQXRZGJFaTKHeP68kMKk3VoVa2b5E1Jjkvy4yTvnG5IwHL215n5+/TlSa5Lcu8kfznViIClUgmGCWqt/Uv39BtJ7jfNWIDJaK39qrtBxj1ba2+edjzAslEJhgmqqo2q6hNVdWJ3vGW3mT5wJ1FVT0lyVpKvdMdbVdVx042KMXHb5H4kwTBZn0xyUpJ7dcc/S/K3U4sGmIQDk2yX5HdJ0lo7K8l9pxkQsHSSYJis9Vtrn0lyS5K01m5KcvN0QwKWsxtba1fdZqxNJRLGyR5pvegJhsm6rqrunu4vxG5T/dv+ZQkM24+q6tlJ5lbVFklemeTbU44JWAqVYJisV2dmV4j7VdVpSY5M8orphgQsZ6/IzF0hb0hyVJKro+0JVnoqwTBZP07yhSTXZ+ZOcV/MTF8wcCfRWrs+yRur6h0zh81dIVmhBtyRMFUqwTBZRyZ5UJJ/SnJIkgdk5lbKwJ1EVT2iqs5JcnaSc6rqB1W1zbTjApZMJRgm66GttS1nHX+9qn48tWiASfhEkpe21r6ZJFX1qCSHJ3nYVKMClkglGCbre91iuCRJVW2fZP4U4wGWv5sXJMBJ0lr7VpKbphgPI7Pg1snTegyVSjBM1jZJvl1Vv+6ON03y0+5Xp621plIEw/eNqvrnzCyKa0n2THJqVW2dJK21700zOGDRJMEwWbtNOwBg4h7e/XnAbcb/NDNJ8eNXbDiMy7Dv2jZNkmCYoNbar6YdAzBZrbXHTTsG4PbTEwwAy0lVHT/tGIBloxIM8P/bu5MXO8ooDOPPq6gYh2icEMV5wKDi0KI4hAgiiW6cUHCjqERdKPgXOCwdNy4U47hQRIyoiElwInFCk5iEJCIuooJulGgciArhuLjVpgmx032rQ6Wt59cUfW/dqrrn9qbf/vp89UlT56iuC1C/hOk9Oa1LjgRLkjR1vui6AEkTYwiWJGmKVNUtXdcgaWJsh5AkqYUkFwH3Accy+L0aBrdAPKHLuiSNzxAsSVI7TwP3ACuBrR3XImmCDMGSJLWzuare7roI9ZcT44ZjCJYkqZ33kzwELAL+Gt3pSnHS7s0QLElSO+c330fG7HOlOGk3ZwiWJKkFV4xT11w2eTjeIk2SpBaSzEzyaJIVzfZIkpld1yVpfIZgSZLaeQb4Dbi+2X4Fnu20Ikk7ZTuEJEntnFhV1455fn+S1Z1Vo36Jd4cYliPBkiS1syXJxaNPmsUztnRYj6QJcCRYkqR27gSeH9MH/DNwU4f1qEfSbJo8Q7AkSe18CTwInAgcBGwGrgLWdlmUpPEZgiVJaud14BdgFfB9x7VImiBDsCRJ7RxdVfO6LkI9Zj/EUJwYJ0lSOx8nOaPrIiRNjiPBkiS1czFwc5KNwF8MxuWqqs7stiz1hSvGDccQLElSO/O7LkDS5BmCJUlqoaq+7boGSZNnCJYkSZrGXDFuOE6MkyRJUu8YgiX1WpKtSVYnWZfklSQzWlzruSTXNY8XJpk9zrFzk1w4xHt8k+TQYWuUJA0YgiX13ZaqOquqTgf+Bu4Y+2KSodrGquq2qtowziFzgUmHYEnaXjrepitDsCRtsxw4qRmlXZ7kDWBDkj2TPJTk8yRrk9wOkIHHk3yV5B3g8NELJfkgyUjzeF6SVUnWJHk3yXEMwvY9zSj0JUkOS/Jq8x6fJ7moOfeQJEuTrE+ykOn9O0eSdhtOjJMk/h3xnQ8sbnadA5xeVRuTLAA2V9V5SfYBPkqyFDgbOBWYDRwBbACe2e66hwFPAXOaa82qqk1JngB+r6qHm+NeBB6rqg+THAMsAU4D7gU+rKoHklwJ3LpLfxCSph//NB6KIVhS3+2bZHXzeDnwNIM2hc+qamOz/3LgzNF+X2AmcDIwB3ipqrYCPyR5bwfXvwBYNnqtqtr0H3VcBszOtmneBybZv3mPa5pz30ry85CfU5I0hiFYUt9tqaqzxu5ogugfY3cBd1XVku2Ou2IK69gDuKCq/txBLZKkKWZPsCTt3BLgziR7ASQ5Jcl+wDLghqZn+Ejg0h2c+ykwJ8nxzbmzmv2/AQeMOW4pcNfokySjwXwZcGOzbz5w8JR9Kkn/C+n4a7oyBEvSzi1k0O+7Ksk64EkG/0l7Dfi6ee0F4JPtT6yqH4EFwKIka4CXm5feBK4enRgH3A2MNBPvNrDtLhX3MwjR6xm0RXy3iz6jJPVKqqrrGiRJkjSEc84dqY8+XdFpDTP2zsqqGum0iCHYEyxJkjRNBZdNHpbtEJIkSeod2yEkSZKmqSSLga6XUv+pquZ1XMOkGYIlSZLUO7ZDSJIkqXcMwZIkSeodQ7AkSZJ6xxAsSZKk3jEES5IkqXf+AX7IUmaEjSmWAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 720x720 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UXHhHwaDRB8f"
      },
      "source": [
        "## 7. Save BERT Classifier <a class=\"anchor\" id=\"section_7\"></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JPJDIf_qREfr"
      },
      "source": [
        "torch.save(bert_classifier.state_dict(), 'bertclassifier.pth')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "S1LhYWdJXPz3"
      },
      "source": [
        "## 8. Load BERT CLassifier <a class=\"anchor\" id=\"section_8\"></a>\n",
        "\n",
        "Before load the BERT classifier, Pytorch requires user to define the same model architecture as the train have done.\n",
        "\n",
        "With the Loaded classifier, one can use it for the inference purpose. Please refer to the [site](https://pytorch.org/tutorials/beginner/saving_loading_models.html) for more detail."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IH2Lqo1pXc9F",
        "outputId": "e62d61e3-6af4-4cad-e8e4-2f54a272f29c"
      },
      "source": [
        "%%time\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from transformers import BertModel\n",
        "\n",
        "# Create the BertClassfier class\n",
        "class BertClassifier(nn.Module):\n",
        "    \"\"\"Bert Model for Classification Tasks.\n",
        "    \"\"\"\n",
        "    def __init__(self, freeze_bert=False):\n",
        "        \"\"\"\n",
        "        @param    bert: a BertModel object\n",
        "        @param    classifier: a torch.nn.Module classifier\n",
        "        @param    freeze_bert (bool): Set `False` to fine-tune the BERT model\n",
        "        \"\"\"\n",
        "        super(BertClassifier, self).__init__()\n",
        "        # Specify hidden size of BERT, hidden size of our classifier, and number of labels\n",
        "        D_in, H, D_out = 768, 100, 2\n",
        "\n",
        "        # Instantiate BERT model\n",
        "        self.bert = BertModel.from_pretrained('bert-base-uncased')\n",
        "\n",
        "        # Instantiate an one-layer feed-forward classifier\n",
        "        self.classifier = nn.Sequential(\n",
        "            nn.Linear(D_in, H),\n",
        "            nn.ReLU(),\n",
        "            nn.Dropout(0.1),\n",
        "            nn.Linear(H, D_out)\n",
        "        )\n",
        "\n",
        "        # Freeze the BERT model\n",
        "        if freeze_bert:\n",
        "            for param in self.bert.parameters():\n",
        "                param.requires_grad = False\n",
        "        \n",
        "    def forward(self, input_ids, attention_mask):\n",
        "        \"\"\"\n",
        "        Feed input to BERT and the classifier to compute logits.\n",
        "        @param    input_ids (torch.Tensor): an input tensor with shape (batch_size,\n",
        "                      max_length)\n",
        "        @param    attention_mask (torch.Tensor): a tensor that hold attention mask\n",
        "                      information with shape (batch_size, max_length)\n",
        "        @return   logits (torch.Tensor): an output tensor with shape (batch_size,\n",
        "                      num_labels)\n",
        "        \"\"\"\n",
        "        # Feed input to BERT\n",
        "        outputs = self.bert(input_ids=input_ids,\n",
        "                            attention_mask=attention_mask)\n",
        "        \n",
        "        # Extract the last hidden state of the token `[CLS]` for classification task\n",
        "        last_hidden_state_cls = outputs[0][:, 0, :]\n",
        "\n",
        "        # Feed input to classifier to compute logits\n",
        "        logits = self.classifier(last_hidden_state_cls)\n",
        "\n",
        "        return logits"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 60 µs, sys: 0 ns, total: 60 µs\n",
            "Wall time: 65.3 µs\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lFCaz_Q6RqnM",
        "outputId": "c67c0d59-0ec1-483b-ecd9-69117439852d"
      },
      "source": [
        "the_model = BertClassifier()\n",
        "the_model.load_state_dict(torch.load('/content/bertclassifier.pth'))\n",
        "the_model.to(device)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<All keys matched successfully>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 70
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w2SYoHxeR5Ox"
      },
      "source": [
        "test_acc,test_real, test_pred = bert_predict(the_model, test_dataloader)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Bo5jscLXTUEx",
        "outputId": "67f0211e-23fb-4a31-9b1f-b1c47ae55ec1"
      },
      "source": [
        "from sklearn.metrics import classification_report\n",
        "#0: peaceful, 1:non_peaceful\n",
        "y_true1 = test_real.cpu().numpy()\n",
        "y_pred1 = test_pred.cpu().numpy()\n",
        "print(classification_report(y_true1, y_pred1))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.98      0.95      0.96     35506\n",
            "           1       0.88      0.95      0.91     14494\n",
            "\n",
            "    accuracy                           0.95     50000\n",
            "   macro avg       0.93      0.95      0.94     50000\n",
            "weighted avg       0.95      0.95      0.95     50000\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pppU97VvXKys"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}