CourseraProject_Sentiment Analysis with Deep Learning use BERT.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "378712fbb3974ee9ac254d3ba1945543": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_567c2c3dcf4e48adbeefa203a2e43784",
              "IPY_MODEL_38040dc8dbfe452b8cbebdef03988d98",
              "IPY_MODEL_8d0e6c0abef54d2d8ce5dc1489e6f654"
            ],
            "layout": "IPY_MODEL_f932fc82aa224e6b8d96b1b97f8d08c8"
          }
        },
        "567c2c3dcf4e48adbeefa203a2e43784": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cd4316c436b44b5eb521162abde5dd90",
            "placeholder": "​",
            "style": "IPY_MODEL_536773c0233b45c08d62e65200d10aad",
            "value": "  0%"
          }
        },
        "38040dc8dbfe452b8cbebdef03988d98": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0d6a5ffbd04c42c69bfcadcf688df16e",
            "max": 5,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0becc2bb02504efabcaa5e4bad8f9531",
            "value": 0
          }
        },
        "8d0e6c0abef54d2d8ce5dc1489e6f654": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4ec4a13536a2405ab363c39a2a7e73dc",
            "placeholder": "​",
            "style": "IPY_MODEL_f15ac715982c4881a5358b094ec5db97",
            "value": " 0/5 [00:00&lt;?, ?it/s]"
          }
        },
        "f932fc82aa224e6b8d96b1b97f8d08c8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cd4316c436b44b5eb521162abde5dd90": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "536773c0233b45c08d62e65200d10aad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0d6a5ffbd04c42c69bfcadcf688df16e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0becc2bb02504efabcaa5e4bad8f9531": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "4ec4a13536a2405ab363c39a2a7e73dc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f15ac715982c4881a5358b094ec5db97": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "076bae934ad44a1d8c1a94e38cc34909": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_dbd355316b4344fda71d3f6450e0da27",
              "IPY_MODEL_9f219ce1fd0349e3889dda16de53f956",
              "IPY_MODEL_8706354136464d1b9d3d60642a09aae9"
            ],
            "layout": "IPY_MODEL_ad91880fb0934d04a9eae7ae68165fd0"
          }
        },
        "dbd355316b4344fda71d3f6450e0da27": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dc352a2841dc42e5883b9363a2dafa71",
            "placeholder": "​",
            "style": "IPY_MODEL_2d85b26ebfbf4ce7aeaf6ebe0cc43ccd",
            "value": "Epoch 1:   2%"
          }
        },
        "9f219ce1fd0349e3889dda16de53f956": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_83b6d1c3db0f42f9be70d8dd8b2b3a55",
            "max": 40,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_75e2004525684177b88362a4a18ffb1f",
            "value": 1
          }
        },
        "8706354136464d1b9d3d60642a09aae9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5c1e3acb73024319bc1faa4860a634e0",
            "placeholder": "​",
            "style": "IPY_MODEL_ed3be13cbb7f417aa79dbf3d6dcb3053",
            "value": " 1/40 [03:56&lt;2:33:50, 236.68s/it, training_loss=0.499]"
          }
        },
        "ad91880fb0934d04a9eae7ae68165fd0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dc352a2841dc42e5883b9363a2dafa71": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2d85b26ebfbf4ce7aeaf6ebe0cc43ccd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "83b6d1c3db0f42f9be70d8dd8b2b3a55": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "75e2004525684177b88362a4a18ffb1f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5c1e3acb73024319bc1faa4860a634e0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ed3be13cbb7f417aa79dbf3d6dcb3053": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Learning Material Guidance:\n",
        "- Hugging Face. https://huggingface.co/docs/transformers/model_doc/bert\n",
        "- Similar Example:https://chriskhanhtran.github.io/_posts/2019-12-25-bert-for-sentiment-analysis/\n",
        "- BERT Fine-Tuning Tutorial with PyTorch. https://mccormickml.com/2019/07/22/BERT-fine-tuning/#41-bertforsequenceclassification\n",
        "- tqdm progress bar: https://zhuanlan.zhihu.com/p/163613814"
      ],
      "metadata": {
        "id": "wtZsgkIKgIIM"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Exploratory Data Analysis & Data Preprocessing\n",
        "- Dataset:https://figshare.com/articles/dataset/smile_annotations_final_csv/3187909/2\n",
        "   - Contains 3085 tweets, 5 emotions named anger, disgust, happiness, surprise, sadness and the 6th label being not-relevant."
      ],
      "metadata": {
        "id": "yLjLT1WugpLL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pip install tqdm"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2Zvh4TcchuX0",
        "outputId": "fccfb50d-0556-46d9-c136-823dfd5e6f14"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (4.64.1)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#import python package we use\n",
        "import torch\n",
        "import pandas as pd\n",
        "from tqdm.notebook import tqdm #a process bar"
      ],
      "metadata": {
        "id": "jDK60uD5g3wO"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#import dataset\n",
        "content=pd.read_csv('/content/smileannotationsfinal.csv',names=['id', 'text', 'category'])"
      ],
      "metadata": {
        "id": "_hP7-G7YiEdu"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "content.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        },
        "id": "agK5-ASsiS0i",
        "outputId": "2a3ac986-0b10-469a-ca62-955fbc904b9a"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                   id                                               text  \\\n",
              "0  611857364396965889  @aandraous @britishmuseum @AndrewsAntonio Merc...   \n",
              "1  614484565059596288  Dorian Gray with Rainbow Scarf #LoveWins (from...   \n",
              "2  614746522043973632  @SelectShowcase @Tate_StIves ... Replace with ...   \n",
              "3  614877582664835073  @Sofabsports thank you for following me back. ...   \n",
              "4  611932373039644672  @britishmuseum @TudorHistory What a beautiful ...   \n",
              "\n",
              "  category  \n",
              "0   nocode  \n",
              "1    happy  \n",
              "2    happy  \n",
              "3    happy  \n",
              "4    happy  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-5a836483-d023-4897-a1e4-18a8e429810c\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>text</th>\n",
              "      <th>category</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>611857364396965889</td>\n",
              "      <td>@aandraous @britishmuseum @AndrewsAntonio Merc...</td>\n",
              "      <td>nocode</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>614484565059596288</td>\n",
              "      <td>Dorian Gray with Rainbow Scarf #LoveWins (from...</td>\n",
              "      <td>happy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>614746522043973632</td>\n",
              "      <td>@SelectShowcase @Tate_StIves ... Replace with ...</td>\n",
              "      <td>happy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>614877582664835073</td>\n",
              "      <td>@Sofabsports thank you for following me back. ...</td>\n",
              "      <td>happy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>611932373039644672</td>\n",
              "      <td>@britishmuseum @TudorHistory What a beautiful ...</td>\n",
              "      <td>happy</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-5a836483-d023-4897-a1e4-18a8e429810c')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-5a836483-d023-4897-a1e4-18a8e429810c button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-5a836483-d023-4897-a1e4-18a8e429810c');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#making the index for looking up to required data\n",
        "content.set_index('id',inplace=True)"
      ],
      "metadata": {
        "id": "yXhLrruniZHR"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "content.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 255
        },
        "id": "0eeqb4oo0VgD",
        "outputId": "e7f3b2df-13d9-45fb-e2c7-50a468bac3f0"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                                 text category\n",
              "id                                                                            \n",
              "611857364396965889  @aandraous @britishmuseum @AndrewsAntonio Merc...   nocode\n",
              "614484565059596288  Dorian Gray with Rainbow Scarf #LoveWins (from...    happy\n",
              "614746522043973632  @SelectShowcase @Tate_StIves ... Replace with ...    happy\n",
              "614877582664835073  @Sofabsports thank you for following me back. ...    happy\n",
              "611932373039644672  @britishmuseum @TudorHistory What a beautiful ...    happy"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8dd5b904-d7ec-426f-a7ec-85be45c0b054\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>category</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>611857364396965889</th>\n",
              "      <td>@aandraous @britishmuseum @AndrewsAntonio Merc...</td>\n",
              "      <td>nocode</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>614484565059596288</th>\n",
              "      <td>Dorian Gray with Rainbow Scarf #LoveWins (from...</td>\n",
              "      <td>happy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>614746522043973632</th>\n",
              "      <td>@SelectShowcase @Tate_StIves ... Replace with ...</td>\n",
              "      <td>happy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>614877582664835073</th>\n",
              "      <td>@Sofabsports thank you for following me back. ...</td>\n",
              "      <td>happy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>611932373039644672</th>\n",
              "      <td>@britishmuseum @TudorHistory What a beautiful ...</td>\n",
              "      <td>happy</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8dd5b904-d7ec-426f-a7ec-85be45c0b054')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-8dd5b904-d7ec-426f-a7ec-85be45c0b054 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-8dd5b904-d7ec-426f-a7ec-85be45c0b054');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#count grouping by the category value\n",
        "content.category.value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UNxSTyJP0tMw",
        "outputId": "23fffda5-ec78-411f-eb1d-82236fe7e373"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "nocode               1572\n",
              "happy                1137\n",
              "not-relevant          214\n",
              "angry                  57\n",
              "surprise               35\n",
              "sad                    32\n",
              "happy|surprise         11\n",
              "happy|sad               9\n",
              "disgust|angry           7\n",
              "disgust                 6\n",
              "sad|disgust             2\n",
              "sad|angry               2\n",
              "sad|disgust|angry       1\n",
              "Name: category, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#remove the labels with \"|\" or \"\\\"\n",
        "content=content[~content.category.str.contains('\\|')] #~:which is not include"
      ],
      "metadata": {
        "id": "HdK30dDe1AHM"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#select ids which are not nocode category\n",
        "content=content[content.category !='nocode'] "
      ],
      "metadata": {
        "id": "1wkWq3tBsFuC"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#checkout the after filtering group counting\n",
        "content.category.value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Fv8vh4Q5sXqT",
        "outputId": "e27e3555-f262-4f1e-b9aa-ae7b1f54cabf"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "happy           1137\n",
              "not-relevant     214\n",
              "angry             57\n",
              "surprise          35\n",
              "sad               32\n",
              "disgust            6\n",
              "Name: category, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#change the label into dictionary\n",
        "#return the unique values of series by order,put labels into a variable called possibleLabels\n",
        "possibleLabels=content.category.unique()"
      ],
      "metadata": {
        "id": "27jN5cUMsnXM"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#set up a label dictionary\n",
        "labelDict={}\n",
        "for index,possibleLabels in enumerate(possibleLabels):\n",
        "  labelDict[possibleLabels]=index"
      ],
      "metadata": {
        "id": "Ma71Ky-buCWa"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#sort it in a new column called 'label',replace label by number\n",
        "content['label']=content.category.replace(labelDict)"
      ],
      "metadata": {
        "id": "VCB7JzpxvTZi"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "content.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 255
        },
        "id": "mmoGleZ6wZr1",
        "outputId": "53e6757f-7450-4759-b62e-63027b5a9fda"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                                 text  \\\n",
              "id                                                                      \n",
              "614484565059596288  Dorian Gray with Rainbow Scarf #LoveWins (from...   \n",
              "614746522043973632  @SelectShowcase @Tate_StIves ... Replace with ...   \n",
              "614877582664835073  @Sofabsports thank you for following me back. ...   \n",
              "611932373039644672  @britishmuseum @TudorHistory What a beautiful ...   \n",
              "611570404268883969  @NationalGallery @ThePoldarkian I have always ...   \n",
              "\n",
              "                   category  label  \n",
              "id                                  \n",
              "614484565059596288    happy      0  \n",
              "614746522043973632    happy      0  \n",
              "614877582664835073    happy      0  \n",
              "611932373039644672    happy      0  \n",
              "611570404268883969    happy      0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6e5612bb-7e5f-4cd7-8b5e-861ce55a0934\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>category</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>614484565059596288</th>\n",
              "      <td>Dorian Gray with Rainbow Scarf #LoveWins (from...</td>\n",
              "      <td>happy</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>614746522043973632</th>\n",
              "      <td>@SelectShowcase @Tate_StIves ... Replace with ...</td>\n",
              "      <td>happy</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>614877582664835073</th>\n",
              "      <td>@Sofabsports thank you for following me back. ...</td>\n",
              "      <td>happy</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>611932373039644672</th>\n",
              "      <td>@britishmuseum @TudorHistory What a beautiful ...</td>\n",
              "      <td>happy</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>611570404268883969</th>\n",
              "      <td>@NationalGallery @ThePoldarkian I have always ...</td>\n",
              "      <td>happy</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6e5612bb-7e5f-4cd7-8b5e-861ce55a0934')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-6e5612bb-7e5f-4cd7-8b5e-861ce55a0934 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-6e5612bb-7e5f-4cd7-8b5e-861ce55a0934');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#look at the category distribution\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "plt.figure(figsize=(4,6))\n",
        "sns.countplot(content.category,\n",
        "              palette='Paired')\n",
        "plt.xlabel('Category')\n",
        "plt.ylabel('Distribution(count)')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 468
        },
        "id": "c-eMM5eRDaqt",
        "outputId": "ba2db47f-733b-4f5d-fe40-787ba33c4894"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/seaborn/_decorators.py:43: FutureWarning: Pass the following variable as a keyword arg: x. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.\n",
            "  FutureWarning\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0, 0.5, 'Distribution(count)')"
            ]
          },
          "metadata": {},
          "execution_count": 17
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 288x432 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAR0AAAFzCAYAAAADlN1RAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAcFklEQVR4nO3de5xcZZ3n8c+Xa4JcwqVFDOwkYtQBVhACgqjLRRAQDTqAMBoCsmYdQUFWBhwdcRldcWUHREecCEyCOiACGQKDYkQCisOlEyAJIhIggWQDNPdrBgK//eN5Gk461Z2qTtVT3dXf9+vVrz7nOafO+VV39bfPeeo8pxQRmJmVsk67CzCzkcWhY2ZFOXTMrCiHjpkV5dAxs6IcOmZW1HrtLqAVttpqqxg3bly7yzAbkebOnft4RHT1t7wjQ2fcuHF0d3e3uwyzEUnSkoGW+/TKzIpy6JhZUQ4dMyvKoWNmRTl0zKwoh46ZFeXQMbOiHDpmVpRDx8yKcuiYWVEOHTMryqFjZkU5dMysqI4cZd7XzLmL2rLfj+/29rbs12wo85GOmRXl0DGzohw6ZlaUQ8fMinLomFlRDh0zK8qhY2ZFOXTMrCiHjpkV5dAxs6IcOmZWlEPHzIpy6JhZUQ4dMyvKoWNmRTl0zKwoh46ZFeXQMbOiHDpmVpRDx8yKcuiYWVEOHTMryqFjZkU5dMysKIeOmRXl0DGzohw6ZlaUQ8fMinLomFlRDh0zK8qhY2ZFOXTMrCiHjpkV1bLQkXSRpMckLay0bSFptqT78vfNc7sknSdpkaT5knatPGZKXv8+SVNaVa+ZldHKI53pwEF92k4Hro+ICcD1eR7gYGBC/poKnA8ppIAzgPcCewBn9AaVmQ1PLQudiLgJeLJP8yRgRp6eARxWab84kluAMZK2AT4MzI6IJyPiKWA2qweZmQ0jpft0to6I5Xn6EWDrPD0WeLiy3tLc1l+7mQ1TbetIjogAolnbkzRVUrek7p6enmZt1syarHToPJpPm8jfH8vty4DtKuttm9v6a19NREyLiIkRMbGrq6vphZtZc5QOnVlA7ztQU4CrKu3H5Hex9gSeyadh1wEHSto8dyAfmNvMbJhar1UblnQJsA+wlaSlpHehzgIuk3Q8sAQ4Mq9+LXAIsAh4ETgOICKelPQPwO15vTMjom/ntJkNIy0LnYg4up9F+9dYN4AT+tnORcBFTSzNzNrIVySbWVEOHTMryqFjZkU5dMysKIeOmRXl0DGzohw6ZlaUQ8fMinLomFlRDh0zK8qhY2ZFOXTMrCiHjpkV5dAxs6IcOmZWlEPHzIpy6JhZUQ4dMyvKoWNmRTl0zKwoh46ZFeXQMbOiHDpmVpRDx8yKcuiYWVEOHTMryqFjZkU5dMysKIeOmRXl0DGzohw6ZlaUQ8fMinLomFlRDh0zK8qhY2ZFOXTMrCiHjpkV5dAxs6IcOmZWlEPHzIpy6JhZUQ4dMyvKoWNmRTl0zKwoh46ZFeXQMbOi2hI6kr4k6W5JCyVdImmUpPGSbpW0SNLPJW2Q190wzy/Ky8e1o2Yza47ioSNpLPBFYGJE7ASsCxwFfAc4JyLeDjwFHJ8fcjzwVG4/J69nZsNUu06v1gNGS1oP2AhYDuwHXJ6XzwAOy9OT8jx5+f6SVLBWM2ui4qETEcuAs4GHSGHzDDAXeDoiVubVlgJj8/RY4OH82JV5/S37blfSVEndkrp7enpa+yTMbNDacXq1OenoZTzwVuBNwEFru92ImBYREyNiYldX19puzsxapB2nVx8CHoyInoh4BbgS2BsYk0+3ALYFluXpZcB2AHn5ZsATZUs2s2ZpR+g8BOwpaaPcN7M/8EfgBuDwvM4U4Ko8PSvPk5f/NiKiYL1m1kTt6NO5ldQhPA9YkGuYBpwGnCJpEanP5sL8kAuBLXP7KcDppWs2s+ZZb82rNF9EnAGc0af5AWCPGuuuAI4oUZeZtZ6vSDazohw6ZlaUQ8fMinLomFlRDh0zK6rud68kTQQ+QLqK+CVgITA7Ip5qUW1m1oHWeKQj6ThJ84CvAKOBe4HHgPcDv5E0Q9J/aW2ZZtYp6jnS2QjYOyJeqrVQ0i7ABNKVxmZmA1pj6ETEPwFI2jsibq4uq9VmZjaQRjqSv19nm5lZv9Z4pCNpL+B9QJekUyqLNiXd9c/MrG719OlsAGyc192k0v4sb4wKNzOrSz19OjcCN0qaHhFLCtRkZh2skVHmG0qaBoyrPi4i9mt2UWbWuRoJnV8APwIuAF5tTTlm1ukaCZ2VEXF+yyoxsxGhkbfMr5b0eUnbSNqi96tllZlZR2rkSKf3PsWnVtoCeFvzyjGzTld36ETE+FYWYmYjQyOjzI+p1R4RFzevHDPrdI2cXu1emR5F+uiYeYBDx8zq1sjp1Req85LGAJc2vSIz62hrc+fAF0gfDWxmVrdG+nSuJr1bBWmg518Cl7WiKDPrXI306ZxdmV4JLImIpU2ux8w6XN2nV3ng559II803B15uVVFm1rnqDh1JRwK3kT7i90jgVkm+tYWZNaSR06uvArtHxGMAkrqA3wCXt6IwM+tMjbx7tU5v4GRPNPh4M7OGjnR+Jek64JI8/0ngl80vycw6WSMXB54q6ROkz7sCmBYRM1tTlpl1qkau0xkPXBsRV+b50ZLGRcTiVhVnZp2nkT6ZXwCvVeZfzW1mZnVrJHTWi4jXr83J0xs0vyQz62SNhE6PpI/1zkiaBDze/JLMrJM18u7V54CfSfpBnl8KTG5+SWbWyRp59+p+YE9JG+f551tWlZl1rDWeXkn6tKTX14uI56uBI2l7Se+v/Wgzs1XVc6SzJXCHpLnAXKCHdOfAtwP/jdSvc3rLKjSzjlLPxwp/L/fj7AfsDbwbeAm4B5gcEQ+1tkQz6yR19elExKvA7PxlZjZojVyR3AV8ltU/y/wzzS/LzDpVI2+ZXwX8jnQ7C3+WuZkNSiOhs1FEnNaySsxsRGjkiuRrJB3SskrMbERoJHROIgXPCknP5a9nB7NTSWMkXS7pT5LukbSXpC0kzZZ0X/6+eV5Xks6TtEjSfEm7DmafZjY0NHJj9k0iYp2IGJWnN4mITQe53+8Bv4qIdwE7k95+Px24PiImANfzxrU/BwMT8tdU4PxB7tPMhoBG+nTIAz4/mGfnRMQ1je5Q0mZ5G8fC66PVX84DSPfJq80A5gCnAZOAiyMigFvyUdI2EbG80X2bWfs18mkQZ5FOsf6Yv06S9O1B7HM86armf5F0h6QLJL0J2LoSJI8AW+fpscDDlccvzW1mNgw10qdzCHBARFwUERcBBwEfGcQ+1wN2Bc6PiPeQPp54lWEU+agmajy2X5KmSuqW1N3T0zOIssyshEY/zWFMZXqzQe5zKbA0Im7N85eTQuhRSdsA5O+9nzyxDNiu8vhtc9sqImJaREyMiIldXV2DLM3MWq2R0Pk2aeDndEkzSIM/v9XoDiPiEeBhSe/MTfuTTtdmAVNy2xTSxYjk9mPyu1h7As+4P8ds+GrkfjqXSJoD7J6bTssBMhhfIN0QbAPgAeA4UgBeJul4YAnpU0QBriWd2i0CXszrmtkwtcbQkfSuiPhT5fqYpfn7WyW9NSLmNbrTiLgTmFhj0f411g3ghEb3YWZDUz1HOqeQro/5vzWWBemWF2ZmdannfjpT8+TBEbGiukzSqJZUZWYdq5GO5D/U2WZm1q96+nTeQroYb7Sk9wDKizYFNmphbWbWgerp0/kwacjCtsA/VtqfA/6uBTWZWQerp09nBjBD0l9FxBUFajKzDtbIgM+dJO3YtzEizmxiPWbW4RoJneqH640CDiXdksLMrG6NXJG8ynU6ks4Grmt6RWbW0Rod8Fm1Ealz2cysbo18BM0C3rjdxLpAF+D+HDNrSCN9OodWplcCj0bEyibXY2YdrpE+nSV50Of7SUc8vwfuaFVhZtaZGrld6ddJ9y7eEtgKmC7pa60qzMw6UyOnV58Cdu4d9JnvmXwn8M1WFGZmnamRd6/+H+n6nF4bUuO2oWZmA6lnwOf3SX04zwB3S5qd5w8AbmtteWbWaeo5verO3+cCMyvtc5pejZl1vHoHfJqZNUU9p1eXRcSRfS4OfF1EvLsllZlZR6rn9Oqk/P3QAdcyM6tDPadXyyWtC0yPiH0L1GRmHayut8wj4lXgNUmD/VRPMzOg8fvpLMhvmb/Q2xgRX2x6VWbWsRoJnSvzV9VqHctmZgNpJHTGRMT3qg2STupvZTOzWhoZBjGlRtuxTarDzEaIeq7TORr4a2C8pFmVRZsCT7aqMDPrTPWcXv0BWE66nUX1PsnPAfNbUZSZda56rtNZAiyR9CHgpYh4TdI7gHcBC1pdoJl1lkb6dG4CRkkaC/wamAxMb0VRZta5GgkdRcSLwCeAH0bEEcBqH75nZjaQhkJH0l6kOwj+e25bt/klmVknayR0Tga+AsyMiLslvQ24oTVlmVmnauTTIG4EbqzMPwB4CISZNaSe63TOjYiTJV1N7fvpfKwllZlZR6rnSOcn+fvZrSzEzEaGeq7TmZu/3yipK0/3tLowM+tMdXUkS/qGpMeBe4E/S+rJH75nZtaQNYaOpFOAvYHdI2KLiNgceC+wt6QvtbpAM+ss9RzpTAaOjogHexvyO1efBo5pVWFm1pnqCZ31I+Lxvo25X2f95pdkZp2sntB5eZDLzMxWU89b5jtLerZGu1j1s83NzNaonrfMPb7KzJqmkbFXZmZrrW2hI2ldSXdIuibPj5d0q6RFkn4uaYPcvmGeX5SXj2tXzWa29tp5pHMScE9l/jvAORHxduAp4PjcfjzwVG4/J69nZsNUW0JH0rbAR4AL8ryA/YDL8yozgMPy9KQ8T16+f17fzIahdh3pnAv8LfBant8SeDoiVub5pcDYPD0WeBggL38mr29mw1Dx0JF0KPBY70DSJm53qqRuSd09PR6PajZUteNIZ2/gY5IWA5eSTqu+B4yR1PsW/rbAsjy9DNgOIC/fDHii70YjYlpETIyIiV1dXa19BmY2aMVDJyK+EhHbRsQ44CjgtxHxKdKtTw/Pq00BrsrTs3jj00UPz+v7M9TNhqmhdJ3OacApkhaR+mwuzO0XAlvm9lOA09tUn5k1Qd33SG6FiJgDzMnTDwB71FhnBXBE0cLMrGWG0pGOmY0ADh0zK8qhY2ZFOXTMrCiHjpkV5dAxs6IcOmZWlEPHzIpy6JhZUQ4dMyvKoWNmRTl0zKwoh46ZFeXQMbOiHDpmVpRDx8yKcuiYWVEOHTMryqFjZkU5dMysKIeOmRXl0DGzohw6ZlaUQ8fMinLomFlRDh0zK8qhY2ZFOXTMrCiHjpkV5dAxs6IcOmZWlEPHzIpy6JhZUQ4dMyvKoWNmRTl0zKwoh46ZFeXQMbOiHDpmVpRDx8yKcuiYWVEOHTMryqFjZkU5dMysKIeOmRW1XrsLGKl2O/Xitux37nePact+zXoVP9KRtJ2kGyT9UdLdkk7K7VtImi3pvvx989wuSedJWiRpvqRdS9dsZs3TjtOrlcD/jIgdgD2BEyTtAJwOXB8RE4Dr8zzAwcCE/DUVOL98yWbWLMVDJyKWR8S8PP0ccA8wFpgEzMirzQAOy9OTgIsjuQUYI2mbwmWbWZO0tSNZ0jjgPcCtwNYRsTwvegTYOk+PBR6uPGxpbuu7ramSuiV19/T0tKxmM1s7bQsdSRsDVwAnR8Sz1WUREUA0sr2ImBYREyNiYldXVxMrNbNmakvoSFqfFDg/i4grc/OjvadN+ftjuX0ZsF3l4dvmNjMbhtrx7pWAC4F7IuIfK4tmAVPy9BTgqkr7MfldrD2BZyqnYWY2zLTjOp29gcnAAkl35ra/A84CLpN0PLAEODIvuxY4BFgEvAgcV7ZcM2um4qETEb8H1M/i/WusH8AJLS3KzIrxMAgzK8qhY2ZFOXTMrCiHjpkV5dAxs6IcOmZWlEPHzIpy6JhZUQ4dMyvKoWNmRTl0zKwoh46ZFeXQMbOiHDpmVpRDx8yKcuiYWVEOHTMryqFjZkU5dMysKIeOmRXl0DGzohw6ZlaUQ8fMinLomFlRDh0zK8qhY2ZFOXTMrCiHjpkV5dAxs6IcOmZWlEPHzIpy6JhZUQ4dMytqvXYXYEPHNXf/sC37PXTHz7dlv9YePtIxs6IcOmZWlEPHzIpy6JhZUQ4dMyvK716ZdYjbJ+7Rlv3u3n1bQ+s7dMwG4elL/rUt+x1z9F+3Zb/N5NMrMyvKoWNmRTl0zKwoh46ZFTVsQkfSQZLulbRI0untrsfMBmdYhI6kdYF/Ag4GdgCOlrRDe6sys8EYLm+Z7wEsiogHACRdCkwC/tjWqqyIfb+/d/F93vCFm4vvc6QYFkc6wFjg4cr80txmZsOMIqLdNayRpMOBgyLiv+f5ycB7I+LEyjpTgal59p3AvU3a/VbA403aVrO4pvoNxbo6vaa/iIiu/hYOl9OrZcB2lfltc9vrImIaMK3ZO5bUHRETm73dteGa6jcU6xrpNQ2X06vbgQmSxkvaADgKmNXmmsxsEIbFkU5ErJR0InAdsC5wUUTc3eayzGwQhkXoAETEtcC1bdh100/ZmsA11W8o1jWiaxoWHclm1jmGS5+OmXWIERE6ksZJWjgE6jhW0lvXchtFn4ukkyVtVGp/A9TxDUlflnSmpA8V2N9hJa96Hyqv0b4kXStpTDO3OSJCZwg5Flhj6EgaSn1tJwNrHTrNek4R8fWI+E0ztrUGh5GG3HSUen8PStaJiEMi4ummFhERHf8FjAPuAX4M3A38GhgNfJb0dvxdwBXARnn96cCPgG7gz8Chuf1Y4CpgDnAfcE7e7jxgeWW7FwCLgfnATGBz4HDgedJFi3cCo/vUuA/wO9KlAH8mvUv33VzffOB/VJ7Lwjzd3zqXAh+pbHt63v+4vI95+et9lX3PAS4H/gT8DPg34CHgNdIV4Dfk+r+Vf163AFvnx2+f5xcA3wSe7+c5nQmcXKnrW8BJA/zevpof93vgEuDLvc8lLz+LNBRmPnB2HbVcU9n2D4Bja20HeB/wJPBg/l1t38Br7U3Av+ef0ULgk8DX8+9oIanDtrcvdbe83l3597hwLfezGNgqL58IzMnT3wB+Atycf47Hsurr+IzKa+te4GLS38lf9G6z1v4qz+FGYC7p3eVt1lh7uwOhYOisBHbJ85cBnwa2rKzzTeALlT/SX5GOBCeQ/uhG5V/WcmBLUrjcm7d7COmP+DJgMvCfwEfzts4Ezs3Tc4CJ/dS4D/ACMD7PTwW+lqc3JAXgeFYNnf7W+TgwI7dvQBpCMpp0xDIqt08Auiv7foZ00eU6wH8AB+dlS0jBuiUQlef1fyr7vgY4Ok9/jlX/0KvPaRwwL0+vA9xf/R30+XnsRgqOjYBNgUVUQifXcy9v/AGPqaOW1UJngO1MJ4dbg6+1vwJ+XJnfDNiiMv+Tys9wPvDBPN1o6NTaz2L6D5255H90rP46XpjXH0f6J7NnZbuLSaFTa3/rA38AunLbJ0mXswxY+0g6vXowIu7M03NJP+CdJP1O0gLgU8COlfUvi4jXIuI+4AHgXbl9dkQ8EREvkYLpqUhv5z9B+kUeCLwaEVfn9WcAH6yzxtsi4sE8fSBwjKQ7gVtJL5AJfdbvb51fAvtK2pA0Mv+mXO/6wI/z8/0Fq54+3BYRSyPiNdJ/989KugvYhjTObQLwMumPuvozBNgrbw+g782DX39OEbEYeELSe3Ltd0TEE/38LD4AzIyIFyPiWVa/GPQZYAVwoaRPAC/WUUst/W1nsBYAB0j6jqQPRMQzpN/Frfnnvh+wY+4nGRMRN+XH/aQJ+xnIrPwa6FV9HV8JvD+3L4mIW+rc3zuBnYDZ+TX4NdI/rgENpb6DVvvPyvSrpISfDhwWEXdJOpb037BX32sJop/2lfn7BcCJwJtJpyEDkvRe4J/z7NeBZ0lHBa+vQjryuq7P48ataZ283hzgw6T/Ppfm5i8BjwI7k440VlQeUv35vIUUwLuQTjuWkY70Xon8L430M6zn9fNCn/kLSP9p3wJcVMfja4p0wegewP6kI58TSX/Q/VnJqn2Yowa5nTXV9WdJu5KOfr8p6XrgBNIR7sOSvtG777XRz36qz7HvPvr+Hvp7ffddb6D9zQTujoi9Gql9JB3p1LIJsFzS+qQjnaojJK0jaXvgbbwxgPQASVtIGk36b937S5pJSv43521+ILdPJp3zAjyX90lE3BoRu+SvWkM6rgP+JteGpHdIelMD6/wcOI50xPCr3LYZsDwfzUwm9QnVsgHwQkS8SDq6WdOYnFtIh9+QhqgMZCZwELB7rr8/NwGHSRotaRPgo9WFkjYGNstHmV8iBelAtSwBdpC0YT7K2H8N23n9d9WI/O7kixHxU9Ip06550eN5X4cDROqcfVpS7xFG39ffYPazmHRaCm/8DPpTfR0fRurvaXR/9wJdkvbK66wvaccBNgOMrCOdWv6edFrSk79XX2QPAbeR+hM+FxErJJHbriAdRl4DHAAQES9Lup90BHAi8KP8VvMDpD9+yB3Ukl4C9upzuNvXBeQ+EKUd95BeHPWu82vSIftVEfFybvshcIWkY0hBVPO/GukPdDtJ95COhl4ldZr352Tgp5K+mrfb76F+/jndADwdEa8OsN48ST8ndVw+RuqIrdoEuErSKNIR3ykD1ZKPMi4j9V88CNyxhu1cSjoV/SKpb+f+AZ5/1X8FvivpNeAV4G9Iv5OFwCN9nsdxwEWSgvT7akSt/YwmnSb+A6n/cCDV1/FPI6K7z1H0GveXf5eHA+dJ2oyUJ+eSOqH75SuSa5A0ndTpeHmf9mNJh8kn1njMOqTO5CNyP9CIkcP1pYgISUeROnIn9bNuS39OjdQyUg30Oi5hpB/pNEW+iOwaUsfniAqcbDfgB/lo62ngM7VWKvRzqqsWax8f6ZhZUSO9I9nMCnPomFlRDh0zK8qhY2tF0lskXSrpfklz86jkd/Sz7hhJny9dow0tDh0btPwO0UzSGJ/tI2I34CvA1v08ZAzQ8tAZYqP0rQ+Hjq2NfUlDI37U2xARdwF3SLpe0jxJCyT1XidzFrC9pDslfRdA0qmSbpc0X9L/6t2OpL9X+hjp30u6RNKXc/sukm7J68+UtHlunyPpXEndwFclPVi5UnvT6ry1l/8j2NrYiTTws68VwMcj4llJWwG3SJoFnA7sFBG7AEg6kDSQdA/S1cCzJH0QeIl0Gf/OpEGq8yr7uZg03uxGSWcCZ5CuQgbYIPLHqOSraz9CukXHUcCVEfFKE5+7DZJDx1pBwP/OAfIaaZR6rVOuA/NX75CEjUkhtAlp+MYKYIWkqwHypfZjIqJ3LNsM3hhRDmm8Wa8LgL8lhc5xpHsn2RDg0LG1cTd5AGMfnwK6gN0i4hVJi6k9slrAtyPin1dplE6usW49Xh9LFhE3K90CdB9g3YgYcrcCHancp2Nr47fAhkof6QyApHeT7jj3WA6cffM8rD5y+zrgM3n0NZLGSnozacTzRyWNyssOBcj3cHmqnxH8tVxMuqfOv6zl87Qm8pGODVoeVPlx4FxJp5H6chaT7lR3ntJNq7pJt0AlIp6QdLPSDch/GRGnSvpL4D/yCP7ngU9HxO25D2g+6f4/C3hj5PoUao/gr+VnpDtCXtLEp21ryWOvbEiStHFEPJ/D5SZgakTMa3AbhwOTImJyS4q0QfGRjg1V0/Ko9FGk+z03GjjfJ92q9ZBWFGeD5yMdMyvKHclmVpRDx8yKcuiYWVEOHTMryqFjZkU5dMysqP8PN6DjpNeTR5AAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#visualize the tweet length\n",
        "plt.figure(figsize=(10, 5))\n",
        "sns.histplot([len(s) for s in content.text], \n",
        "             bins=100)\n",
        "plt.title('Tweet Length')\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 336
        },
        "id": "yKvJDYPHE-53",
        "outputId": "a306fdbb-c1a7-4beb-bca8-114f42202940"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 720x360 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmQAAAE/CAYAAADosN8VAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAck0lEQVR4nO3df/RldV3v8ecLJlBQRJyJYAYaTLLMa8oaibKVXvEWFom1uIDLcjRoriszKEtBXdr1ZpdurhRvN20EA7sufkR4xVKLyLBWAQ3QADKUc0FgpgEGFfHmr0bf94+zkTPD+c6cme/3nM/58Xys9V3fsz97n3PeZ8+e7/f1/Xw+e+9UFZIkSWpnv9YFSJIkzTsDmSRJUmMGMkmSpMYMZJIkSY0ZyCRJkhozkEmSJDVmIJOkCZTkN5P879Z1SBoPA5mksUvy//q+vpXkq33Lrxzxe38uyUt2s/5FSbaMsoZJeE9Jk2VZ6wIkzZ+qetKjj5N8Djirqv6qXUWS1JY9ZJImQpIndD1ly7vltyTZkeSQbvm/JXlP9/jAJO9Kcm+SB5K8P8kT+17r5CT/lOThJH+f5Dld+x8DRwMf63rj3riXNR6Z5E+TbE9yd5Jf6Vv3m0muSPKhJF9O8pkka/rWH5fklm7dnyS5PMlvJTkY+ARwZF8v4ZHd0w5Y6PUkzRYDmaSJUFVfA/4ReGHX9ELgHuAFfcvXdY/PB74XeC7wDGAl8DaAJM8DPgj8F+BpwB8CVyc5sKp+HrgX+OmqelJV/Y9h60uyH/AxYGP3ficC5yT5ib7NXgZcBhwKXA38fvfcA4CPABcDhwGXAj/Tfe5/A14K/GtX05Oq6l9393qSZo+BTNIkuQ54YZJlwHOA93bLTwCeD3w6SYB1wK9W1Req6svAbwNndK+xDvjDqrqhqr5ZVZcAXwdOWGRtzwdWVNU7quobVXUX8IG+9wX4u6r6eFV9E/hj4Ae79hPoTRF5b1X9e1VdBdw4xHsu9HqSZoxzyCRNkuuA3wOOA24DrgEuohdoNlfV55N8J3AQcFMvmwEQYP/u8XcDa5O8vu91DwCOZHG+m96w4sN9bfsDf9u3fH/f468AT+jC5ZHA1qqqvvX3DfGeA1+vqnbsXemSJp2BTNIk+XvgmfSG866rqjuSHA38JI8NVz4EfBX4garaOuA17gPeWVXvXOA9aoH2PbkPuLuqjt2H524DViZJXyg7Cvi/i6xJ0oxwyFLSxKiqrwA3Aa/jsQD298BrH12uqm/RGyp8d9dbRpKVfXO5PgC8NskPpefgJD+V5Mnd+geAp++plu4kg29/0Rti/HKSNyV5YpL9kzw7yfOH+Gj/AHwT+OUky5KcAhzft/4B4GlJnjLEa0maQQYySZPmOuA7eGyO1XXAk4FP923zJmAzcH2SR4C/otezRlVtAH6R3gT4L3bbvbrvuf8deGt3BuavL1DDSnq9cP1fxwAn0zuR4G56PXUXAnsMUVX1DeBngTOBh4GfA/6M3tw2qupOehP97+rqWuzwqqQpk52nNEiSxiHJDcD7q+qPWtciqT17yCRpDJK8MMl3dUOWa+mdRfrJ1nVJmgxO6pek8XgmcAVwMHAXcGpVbWtbkqRJ4ZClJElSYw5ZSpIkNWYgkyRJamyq55AtX768Vq9e3boMSZKkPbrpppseqqoVg9ZNdSBbvXo1GzZsaF2GJEnSHiW5Z6F1DllKkiQ1ZiCTJElqzEAmSZLUmIFMkiSpMQOZJElSYwYySZKkxgxkkiRJjY0skCX5YJIHk9w+YN0bklSS5d1ykrw3yeYktyY5blR1SZIkTZpR9pBdDJy0a2OSo4AfB+7ta34pcGz3tQ543wjrkiRJmigjC2RV9WngCwNWvRt4I1B9bacAH6qe64FDkxwxqtokSZImyVjnkCU5BdhaVRt3WbUSuK9veUvXJkmSNPPGdi/LJAcBb6Y3XLmY11lHb1iTo48+egkqkyRJS+Xsc9/G1oce2alt5fJDuOD8dzSqaDqM8+bi3wMcA2xMArAKuDnJ8cBW4Ki+bVd1bY9TVeuB9QBr1qypQdtIkqQ2tj70CMuOP33nthsvb1TN9BjbkGVV3VZV31lVq6tqNb1hyeOq6n7gauBV3dmWJwBfqqpt46pNkiSppVFe9uJS4B+AZybZkuTM3Wz+ceAuYDPwAeCXRlWXJEnSpBnZkGVVvWIP61f3PS7gdaOqRZIkaZJ5pX5JkqTGDGSSJEmNGcgkSZIaM5BJkiQ1ZiCTJElqzEAmSZLUmIFMkiSpMQOZJElSYwYySZKkxgxkkiRJjRnIJEmSGjOQSZIkNWYgkyRJasxAJkmS1JiBTJIkqTEDmSRJUmMGMkmSpMYMZJIkSY0ZyCRJkhozkEmSJDVmIJMkSWrMQCZJktSYgUySJKkxA5kkSVJjBjJJkqTGDGSSJEmNGcgkSZIaM5BJkiQ1NrJAluSDSR5Mcntf2+8muTPJrUk+kuTQvnXnJdmc5J+T/MSo6pIkSZo0o+whuxg4aZe2a4BnV9VzgH8BzgNI8izgDOAHuuf8QZL9R1ibJEnSxBhZIKuqTwNf2KXtL6tqR7d4PbCqe3wKcFlVfb2q7gY2A8ePqjZJkqRJ0nIO2S8An+gerwTu61u3pWuTJEmaeU0CWZK3ADuAD+/Dc9cl2ZBkw/bt25e+OEmSpDEbeyBL8mrgZOCVVVVd81bgqL7NVnVtj1NV66tqTVWtWbFixUhrlSRJGoexBrIkJwFvBF5WVV/pW3U1cEaSA5McAxwL3DjO2iRJklpZNqoXTnIp8CJgeZItwNvpnVV5IHBNEoDrq+q1VfWZJFcAd9AbynxdVX1zVLVJkiRNkpEFsqp6xYDmi3az/TuBd46qHkmSpEnllfolSZIaM5BJkiQ1ZiCTJElqzEAmSZLUmIFMkiSpMQOZJElSYwYySZKkxgxkkiRJjRnIJEmSGjOQSZIkNWYgkyRJasxAJkmS1JiBTJIkqTEDmSRJUmMGMkmSpMYMZJIkSY0ZyCRJkhozkEmSJDVmIJMkSWrMQCZJktSYgUySJKkxA5kkSVJjBjJJkqTGDGSSJEmNGcgkSZIaM5BJkiQ1ZiCTJElqzEAmSZLU2MgCWZIPJnkwye19bYcluSbJZ7vvT+3ak+S9STYnuTXJcaOqS5IkadKMsofsYuCkXdrOBa6tqmOBa7tlgJcCx3Zf64D3jbAuSZKkiTKyQFZVnwa+sEvzKcAl3eNLgJf3tX+oeq4HDk1yxKhqkyRJmiTjnkN2eFVt6x7fDxzePV4J3Ne33ZauTZIkaeY1m9RfVQXU3j4vybokG5Js2L59+wgqkyRJGq9xB7IHHh2K7L4/2LVvBY7q225V1/Y4VbW+qtZU1ZoVK1aMtFhJkqRxGHcguxpY2z1eC3y0r/1V3dmWJwBf6hvalCRJmmnLRvXCSS4FXgQsT7IFeDtwPnBFkjOBe4DTus0/DvwksBn4CvCaUdUlSZI0aUYWyKrqFQusOnHAtgW8blS1SJIkTTKv1C9JktSYgUySJKkxA5kkSVJjBjJJkqTGDGSSJEmNGcgkSZIaM5BJkiQ1ZiCTJElqbGQXhpUkSQK47daNnHrWOTu1rVx+CBec/45GFU0eA5kkSRqpr9V+LDv+9J3att54eaNqJpOBTJIkjZ29ZjszkEmSpLGz12xnTuqXJElqzEAmSZLUmIFMkiSpMQOZJElSYwYySZKkxgxkkiRJjRnIJEmSGjOQSZIkNWYgkyRJasxAJkmS1JiBTJIkqTEDmSRJUmMGMkmSpMYMZJIkSY0ZyCRJkhozkEmSJDXWJJAl+dUkn0lye5JLkzwhyTFJbkiyOcnlSQ5oUZskSdK4jT2QJVkJ/AqwpqqeDewPnAH8DvDuqnoG8EXgzHHXJkmS1EKrIctlwBOTLAMOArYBLwau7NZfAry8UW2SJEljNVQgS/KCYdqGUVVbgXcB99ILYl8CbgIerqod3WZbgJX78vqSJEnTZtgesv85ZNseJXkqcApwDHAkcDBw0l48f12SDUk2bN++fV9KkCRJmijLdrcyyQ8DPwKsSPJrfasOoTf3a1+8BLi7qrZ373EV8ALg0CTLul6yVcDWQU+uqvXAeoA1a9bUPtYgSZI0MfbUQ3YA8CR6we3JfV+PAKfu43veC5yQ5KAkAU4E7gA+1feaa4GP7uPrS5IkTZXd9pBV1XXAdUkurqp7luINq+qGJFcCNwM7gFvo9Xj9OXBZkt/q2i5aiveTJEmadLsNZH0OTLIeWN3/nKp68b68aVW9HXj7Ls13Acfvy+tJkiRNs2ED2Z8A7wcuBL45unIkSZLmz7CBbEdVvW+klUiSJM2pYS978bEkv5TkiCSHPfo10sokSZLmxLA9ZGu777/R11bA05e2HEmSpPkzVCCrqmNGXYgkSdK8GiqQJXnVoPaq+tDSliNJkjR/hh2yfH7f4yfQu5jrzYCBTJIkaZGGHbJ8ff9ykkOBy0ZSkSRJ0pwZ9izLXf0bvZuDS5IkaZGGnUP2MXpnVULvpuLfD1wxqqIkSZLmybBzyN7V93gHcE9VbRlBPZIkSXNnqCHL7ibjdwJPBp4KfGOURUmSJM2ToQJZktOAG4H/DJwG3JDk1FEWJkmSNC+GHbJ8C/D8qnoQIMkK4K+AK0dVmCRJ0rwY9izL/R4NY53P78VzJUmStBvD9pB9MslfAJd2y6cDHx9NSZIkSfNlt4EsyTOAw6vqN5L8LPCj3ap/AD486uIkSZLmwZ56yN4DnAdQVVcBVwEk+Q/dup8eaXWSJElzYE/zwA6vqtt2bezaVo+kIkmSpDmzp0B26G7WPXEpC5EkSZpXewpkG5L84q6NSc4CbhpNSZIkSfNlT3PIzgE+kuSVPBbA1gAHAD8zysIkSZLmxW4DWVU9APxIkv8IPLtr/vOq+uuRVyZJkjQnhroOWVV9CvjUiGuRJEmaS15tX5IkqTEDmSRJUmMGMkmSpMYMZJIkSY01CWRJDk1yZZI7k2xK8sNJDktyTZLPdt+f2qI2SZKkcWvVQ3YB8Mmq+j7gB4FNwLnAtVV1LHBttyxJkjTzxh7IkjwF+DHgIoCq+kZVPQycAlzSbXYJ8PJx1yZJktRCix6yY4DtwB8luSXJhUkOpncj823dNvcDhzeoTZIkaeyGujDsCN7zOOD1VXVDkgvYZXiyqipJDXpyknXAOoCjjz561LVKkjR3zj73bWx96JGd2lYuP4QLzn9Ho4pmX4tAtgXYUlU3dMtX0gtkDyQ5oqq2JTkCeHDQk6tqPbAeYM2aNQNDmyRJ2ndbH3qEZcefvnPbjZc3qmY+jH3IsqruB+5L8syu6UTgDuBqYG3Xthb46LhrkyRJaqFFDxnA64EPJzkAuAt4Db1weEWSM4F7gNMa1SZJkjRWTQJZVf0TsGbAqhPHXYskSVJrXqlfkiSpMQOZJElSYwYySZKkxgxkkiRJjRnIJEmSGjOQSZIkNWYgkyRJasxAJkmS1JiBTJIkqTEDmSRJUmMGMkmSpMYMZJIkSY0ZyCRJkhpb1roASZK09M4+921sfeiRndpWLj+EC85/R6OKtDsGMkmSZtDWhx5h2fGn79x24+WNqtGeOGQpSZLUmIFMkiSpMQOZJElSYwYySZKkxgxkkiRJjRnIJEmSGjOQSZIkNWYgkyRJasxAJkmS1JiBTJIkqTEDmSRJUmPey1KSJO2TQTcwv33TnTz3+EYFTTEDmSRJ2ieDbmD+1Y1vbVTNdGs2ZJlk/yS3JPmzbvmYJDck2Zzk8iQHtKpNkiRpnFrOITsb2NS3/DvAu6vqGcAXgTObVCVJkjRmTQJZklXATwEXdssBXgxc2W1yCfDyFrVJkiSNW6sesvcAbwS+1S0/DXi4qnZ0y1uAlS0KkyRJGrexB7IkJwMPVtVN+/j8dUk2JNmwffv2Ja5OkiRp/Fr0kL0AeFmSzwGX0RuqvAA4NMmjZ32uArYOenJVra+qNVW1ZsWKFeOoV5IkaaTGHsiq6ryqWlVVq4EzgL+uqlcCnwJO7TZbC3x03LVJkiS1MElX6n8T8GtJNtObU3ZR43okSZLGoumFYavqb4C/6R7fBXhtX0mSNHcmqYdMkiRpLhnIJEmSGjOQSZIkNWYgkyRJaqzppH5JkjQdbrt1I6eedc5ObbdvupPnejrekjCQSZKkPfpa7cey40/fqe2rG9/aqJrZ45ClJElSYwYySZKkxhyylCRpQp197tvY+tAjO7WtXH4IF5z/jkYVaVQMZJIkTaitDz3yuHlbW2+8vFE1GiUDmSRJY2SvlwYxkEmSNEb2emkQJ/VLkiQ1ZiCTJElqzEAmSZLUmHPIJElzycn1miQGMknSXHJyvSaJQ5aSJEmNGcgkSZIac8hSkqQ5cdutGzn1rHN2anPe3GQwkEmSNCe+Vvs5b25COWQpSZLUmIFMkiSpMYcsJUnaS17DTEvNQCZJWpR5DCdew0xLzUAmSVoUw4m0eAYySdLUmsfeuUEG7YfbN93Jc49vVNAYzNq/vYFMkjS17J3rGbQfvrrxrY2qGY9Z+7cfeyBLchTwIeBwoID1VXVBksOAy4HVwOeA06rqi+OuT5Lm0az1NkjTpkUP2Q7gDVV1c5InAzcluQZ4NXBtVZ2f5FzgXOBNDeqTpLkza70N0rQZ+3XIqmpbVd3cPf4ysAlYCZwCXNJtdgnw8nHXJkmS1ELTC8MmWQ08D7gBOLyqtnWr7qc3pClJkjTzmk3qT/Ik4E+Bc6rqkSTfXldVlaQWeN46YB3A0UcfPY5SJWlqOTdMmg5NAlmS76AXxj5cVVd1zQ8kOaKqtiU5Anhw0HOraj2wHmDNmjUDQ5skqce5YdJ0aHGWZYCLgE1V9Xt9q64G1gLnd98/Ou7aJEnjZy+e1KaH7AXAzwO3Jfmnru3N9ILYFUnOBO4BTmtQmyRpzOzFkxoEsqr6OyALrD5xnLVImi+L6YmxF0d7ctutGzn1rHN2avMY2TvzvA+9Ur+kiTUoBN312U08/djv36lt2B/Yi+mJWcxzB30OmJ9fNPPia7WfPX2LNM/70EAmaWINCkGf3/hWvnfKfmAP+hww+XVLGh8DmaQlYS+QpFEYNIwJs3fzdAOZpCVhL5CkURg0jAmzd/P0plfqlyRJkoFMkiSpOYcspQV4mQNNCo9FjdKgOVqzNj9rGhjIpAV4sUpNCo9FjdKgOVqzNj9rGhjIJGkKzfMFNGfR3pxJaI/WbDKQSdIUmucLaM6ivTmT0B6t2WQgkzTXlrqnadB8r1nqvZiGz2fvoaaRgUxTyUnO2pNhg8NS9zQNmu+1UO/FNAaHvfl8i7GYfTNJvYcOL2pYBjJNJSc5a0/GFRwWY5KCw6SZlX3j8KKGZSCTpH00670fs/T5ZumzaDYZyCRpH81678csfb5Z+iyaTQYyaYyGnfu21NtNmkmvu2Vvyiz35Cx0A/phP98s7xvJQCaN0bBz35Z6u0kz6XW37E2Z5Z6chW5AP+znm+V9IxnIpL2w0MUbJ713Z6l74e767Caefuz379TWsqfCnpO94/5a2DSe+arZYCCT9sJCF2+c9N6dpe6F+/zGt/K9E9RTYc/J3nF/LWxWzu7U9DGQaaa1nKs0DRfQnCT22iye+1CaXgYyzbSWc5Wm4TpYk8Rem8VzH0rTy0A2AxbTC7TYHqSlfu9Bc5OmYf7GOHompvU97LWZT7P07z5Ln0WTy0A2AxbTC7TYHqSlfu9Bc5OmYf7GOHompvU97LWZT7P07z5Ln0WTy0A2wSb9Wk3jstR/nXoWlSTNj2n5XWogmxALTgBf+/ad2hbTWzSuSeZLHXiG/et02OA26PU+eeGbZ3pIwiEXSfNq0GjMoJ/5rUOagWxCjGMC+Lgmmbc6bXwxwwqzPiQx659PkvbGJF7exEC2jya9C3QxPSILXfx00i4GKknSrDCQ7aNZvvXLQhc/nbSLgUqSNCsmLpAlOQm4ANgfuLCqzm9c0kSZ1rlA01q3JGl6TPPvmokKZEn2B/4X8J+ALcA/Jrm6qu5oW9nkmNa5QNNatyRpekzz75qJCmTA8cDmqroLIMllwClA00C21Gcneksd9Zvmv+gkSUtj0gLZSuC+vuUtwA81quXblvrsRG+po37T/BedJGlppKpa1/BtSU4FTqqqs7rlnwd+qKp+uW+bdcC6bvGZwD+PvdClsxx4qHURU8J9NTz31d5xfw3PfTU899Xw5mlffXdVrRi0YtJ6yLYCR/Utr+ravq2q1gPrx1nUqCTZUFVrWtcxDdxXw3Nf7R331/DcV8NzXw3PfdWzX+sCdvGPwLFJjklyAHAGcHXjmiRJkkZqonrIqmpHkl8G/oLeZS8+WFWfaVyWJEnSSE1UIAOoqo8DH29dx5jMxNDrmLivhue+2jvur+G5r4bnvhqe+4oJm9QvSZI0jyZtDpkkSdLcMZCNSZKjknwqyR1JPpPk7K79sCTXJPls9/2prWudBEn2T3JLkj/rlo9JckOSzUku7076EJDk0CRXJrkzyaYkP+xxNViSX+3+/92e5NIkT/DY6knywSQPJrm9r23gcZSe93b77NYkx7WrfPwW2Fe/2/0fvDXJR5Ic2rfuvG5f/XOSn2hTdTuD9lffujckqSTLu+W5PbYMZOOzA3hDVT0LOAF4XZJnAecC11bVscC13bLgbGBT3/LvAO+uqmcAXwTObFLVZLoA+GRVfR/wg/T2m8fVLpKsBH4FWFNVz6Z34tAZeGw96mLgpF3aFjqOXgoc232tA943phonxcU8fl9dAzy7qp4D/AtwHkD3c/4M4Ae65/xBd5vAeXIxj99fJDkK+HHg3r7muT22DGRjUlXbqurm7vGX6f3SXEnv1lCXdJtdAry8TYWTI8kq4KeAC7vlAC8Gruw2cT91kjwF+DHgIoCq+kZVPYzH1UKWAU9Msgw4CNiGxxYAVfVp4Au7NC90HJ0CfKh6rgcOTXLEeCptb9C+qqq/rKod3eL19K6jCb19dVlVfb2q7gY207tN4NxY4NgCeDfwRqB/MvvcHlsGsgaSrAaeB9wAHF5V27pV9wOHNyprkryH3n/Sb3XLTwMe7vtht4VemBUcA2wH/qgb4r0wycF4XD1OVW0F3kXvr/FtwJeAm/DY2p2FjqNBt7lzvz3mF4BPdI/dVwMkOQXYWlUbd1k1t/vLQDZmSZ4E/ClwTlXtdIfx6p3yOtenvSY5GXiwqm5qXcuUWAYcB7yvqp4H/Bu7DE96XPV0859OoRdijwQOZsAwigbzOBpOkrfQm6Ly4da1TKokBwFvBt7WupZJYiAboyTfQS+MfbiqruqaH3i0O7b7/mCr+ibEC4CXJfkccBm94aQL6HVbP3rdvMfdUmuObQG2VNUN3fKV9AKax9XjvQS4u6q2V9W/A1fRO948tha20HG0x9vczaMkrwZOBl5Zj11Tyn31eN9D7w+jjd3P+lXAzUm+izneXwayMenmQV0EbKqq3+tbdTWwtnu8FvjouGubJFV1XlWtqqrV9CbC/nVVvRL4FHBqt9nc76dHVdX9wH1Jntk1nQjcgcfVIPcCJyQ5qPv/+Oi+8tha2ELH0dXAq7oz4k4AvtQ3tDmXkpxEb6rFy6rqK32rrgbOSHJgkmPoTVa/sUWNk6Kqbquq76yq1d3P+i3Acd3Ps7k9trww7Jgk+VHgb4HbeGxu1JvpzSO7AjgauAc4raoGTX6cO0leBPx6VZ2c5On0eswOA24Bfq6qvt6yvkmR5Ln0ToA4ALgLeA29P7Y8rnaR5L8Cp9MbUroFOIve/JS5P7aSXAq8CFgOPAC8Hfg/DDiOukD7+/SGfL8CvKaqNrSou4UF9tV5wIHA57vNrq+q13bbv4XevLId9KarfGLX15xlg/ZXVV3Ut/5z9M5+fmiejy0DmSRJUmMOWUqSJDVmIJMkSWrMQCZJktSYgUySJKkxA5kkSVJjBjJJkqTGDGSSJEmNGcgkSZIa+//oj4zh5N4VxAAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Training/Validation Split"
      ],
      "metadata": {
        "id": "n3lmD1u3weRi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split"
      ],
      "metadata": {
        "id": "t3TpUsFlwiGX"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X_train,X_val,y_train,y_val=train_test_split(content.index.values,\n",
        "                                             content.label.values,\n",
        "                                             test_size=0.15,\n",
        "                                             random_state=17,\n",
        "                                             stratify=content.label.values)"
      ],
      "metadata": {
        "id": "Tcpl82fRwsSp"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#give the name of datatype\n",
        "content['data_type'] = ['not_set']*content.shape[0] #first fill in with not_set temporarily\n",
        "#fill the datatype with train/val\n",
        "content.loc[X_train,'data_type']='train'\n",
        "content.loc[X_val,'data_type']='val'\n",
        "#group by and count\n",
        "content.groupby(['category','label','data_type']).count()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 492
        },
        "id": "yrMpc3-zzkD0",
        "outputId": "38ed4069-39f8-4a64-fe85-5da3205eeb4d"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                              text\n",
              "category     label data_type      \n",
              "angry        2     train        48\n",
              "                   val           9\n",
              "disgust      3     train         5\n",
              "                   val           1\n",
              "happy        0     train       966\n",
              "                   val         171\n",
              "not-relevant 1     train       182\n",
              "                   val          32\n",
              "sad          4     train        27\n",
              "                   val           5\n",
              "surprise     5     train        30\n",
              "                   val           5"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-275063d0-b10c-4b78-8838-d499feb3c80f\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>category</th>\n",
              "      <th>label</th>\n",
              "      <th>data_type</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">angry</th>\n",
              "      <th rowspan=\"2\" valign=\"top\">2</th>\n",
              "      <th>train</th>\n",
              "      <td>48</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>val</th>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">disgust</th>\n",
              "      <th rowspan=\"2\" valign=\"top\">3</th>\n",
              "      <th>train</th>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>val</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">happy</th>\n",
              "      <th rowspan=\"2\" valign=\"top\">0</th>\n",
              "      <th>train</th>\n",
              "      <td>966</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>val</th>\n",
              "      <td>171</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">not-relevant</th>\n",
              "      <th rowspan=\"2\" valign=\"top\">1</th>\n",
              "      <th>train</th>\n",
              "      <td>182</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>val</th>\n",
              "      <td>32</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">sad</th>\n",
              "      <th rowspan=\"2\" valign=\"top\">4</th>\n",
              "      <th>train</th>\n",
              "      <td>27</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>val</th>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">surprise</th>\n",
              "      <th rowspan=\"2\" valign=\"top\">5</th>\n",
              "      <th>train</th>\n",
              "      <td>30</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>val</th>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-275063d0-b10c-4b78-8838-d499feb3c80f')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-275063d0-b10c-4b78-8838-d499feb3c80f button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-275063d0-b10c-4b78-8838-d499feb3c80f');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Loading the Tokenizer"
      ],
      "metadata": {
        "id": "MuH0ueLgCVwL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pip install transformers"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "naYc-Vl2Rz-T",
        "outputId": "26f3c82d-f72f-4c9f-aa4a-c3a1b35cf50c"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.23.1)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.13.1)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.13.0)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.10.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.10.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2022.6.2)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.8.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.10.0->transformers) (4.1.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.9.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2022.9.24)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pip install transformers[torch]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WTQ-qVoj7bPf",
        "outputId": "fdbcd2d1-2a7f-4bff-d2ae-a907a73d6990"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: transformers[torch] in /usr/local/lib/python3.7/dist-packages (4.23.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers[torch]) (2.23.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers[torch]) (4.13.0)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.10.0 in /usr/local/lib/python3.7/dist-packages (from transformers[torch]) (0.10.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers[torch]) (2022.6.2)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers[torch]) (4.64.1)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers[torch]) (21.3)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.7/dist-packages (from transformers[torch]) (0.13.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers[torch]) (3.8.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers[torch]) (6.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers[torch]) (1.21.6)\n",
            "Requirement already satisfied: torch!=1.12.0,>=1.7 in /usr/local/lib/python3.7/dist-packages (from transformers[torch]) (1.12.1+cu113)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.10.0->transformers[torch]) (4.1.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers[torch]) (3.0.9)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers[torch]) (3.9.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers[torch]) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers[torch]) (2022.9.24)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers[torch]) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers[torch]) (1.24.3)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import BertTokenizer"
      ],
      "metadata": {
        "id": "IGpC8No678QH"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased',  #instantiate a pre-trained model from a pre-trained model configuration.\n",
        "                                          do_lower_case=True)   #lowercases the input."
      ],
      "metadata": {
        "id": "SUv9Jutu8CnR"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Encoding the Data"
      ],
      "metadata": {
        "id": "6ZlGocJ38bHu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#encode training dataset\n",
        "#reference:https://huggingface.co/docs/transformers/v4.23.1/en/main_classes/tokenizer#transformers.PreTrainedTokenizer\n",
        "encoded_data_train=tokenizer.batch_encode_plus(\n",
        "    content[content.data_type=='train'].text.values,      #specify the train set\n",
        "    add_special_tokens=True, #adds the special [CLS] and [SEP] tokens to the beginning and the end of the sentences, respectively.\n",
        "    return_attention_mask=True, #creates attention mask by creating array of 0s (pad token) and 1s (real token).\n",
        "    pad_to_max_length=True, #creates constant length for all sequences by padding to max length.\n",
        "    max_length=256,\n",
        "    return_tensors='pt' #returns PyTorch tensor\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5O-3tsnp8F-r",
        "outputId": "57e833ad-ddc4-4441-eaef-21bd98d319ad"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2308: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#encode validation dataset\n",
        "encoded_data_val=tokenizer.batch_encode_plus(\n",
        "    content[content.data_type=='val'].text.values,\n",
        "    add_special_tokens = True,\n",
        "    return_attention_mask = True,\n",
        "    pad_to_max_length = True,\n",
        "    max_length = 256,\n",
        "    return_tensors = 'pt'\n",
        ")"
      ],
      "metadata": {
        "id": "uXJeEhws0UaP"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "encoded_data_train"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P_v31_rHrTQg",
        "outputId": "471f480b-b675-4eab-ce87-f9950ea0a134"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input_ids': tensor([[  101, 16092,  3897,  ...,     0,     0,     0],\n",
              "        [  101,  1030, 27034,  ...,     0,     0,     0],\n",
              "        [  101,  1030, 10682,  ...,     0,     0,     0],\n",
              "        ...,\n",
              "        [  101, 11047,  1030,  ...,     0,     0,     0],\n",
              "        [  101,  1030,  3680,  ...,     0,     0,     0],\n",
              "        [  101,  1030,  2120,  ...,     0,     0,     0]]), 'token_type_ids': tensor([[0, 0, 0,  ..., 0, 0, 0],\n",
              "        [0, 0, 0,  ..., 0, 0, 0],\n",
              "        [0, 0, 0,  ..., 0, 0, 0],\n",
              "        ...,\n",
              "        [0, 0, 0,  ..., 0, 0, 0],\n",
              "        [0, 0, 0,  ..., 0, 0, 0],\n",
              "        [0, 0, 0,  ..., 0, 0, 0]]), 'attention_mask': tensor([[1, 1, 1,  ..., 0, 0, 0],\n",
              "        [1, 1, 1,  ..., 0, 0, 0],\n",
              "        [1, 1, 1,  ..., 0, 0, 0],\n",
              "        ...,\n",
              "        [1, 1, 1,  ..., 0, 0, 0],\n",
              "        [1, 1, 1,  ..., 0, 0, 0],\n",
              "        [1, 1, 1,  ..., 0, 0, 0]])}"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Encoding:encoded_data_train and encoded_data_val are encoded train set and encoded val set\n",
        "#train set\n",
        "input_ids_train=encoded_data_train['input_ids'] #input_ids: List of token ids to be fed to a model.\n",
        "attention_masks_train=encoded_data_train['attention_mask'] #attention_mask:List of indices specifying which tokens should be attended to by the model\n",
        "labels_train = torch.tensor(content[content.data_type =='train'].label.values) #convert labels to tensor"
      ],
      "metadata": {
        "id": "CXjANOgr07sN"
      },
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#validation set\n",
        "input_ids_val = encoded_data_val['input_ids']\n",
        "attention_masks_val = encoded_data_val['attention_mask']\n",
        "labels_val = torch.tensor(content[content.data_type == 'val'].label.values)"
      ],
      "metadata": {
        "id": "9DoPn9KP2413"
      },
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Build the BERT Model"
      ],
      "metadata": {
        "id": "XcUYxag33DmS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#Set up the Pre-trained Model\n",
        "#references:https://huggingface.co/docs/transformers/v4.23.1/en/main_classes/model#transformers.PreTrainedModel.from_pretrained\n",
        "#https://huggingface.co/docs/transformers/v4.23.1/en/model_doc/bert#transformers.BertForSequenceClassification\n",
        "from transformers import BertForSequenceClassification\n",
        "Bert=BertForSequenceClassification.from_pretrained('bert-base-uncased', #the specific model path/name\n",
        "                                                   num_labels=len(labelDict), #number of category\n",
        "                                                   output_attentions=False, # Whether or not to return the attentions tensors of all attention layers\n",
        "                                                   output_hidden_states=False #Whether or not to return the hidden states of all layers\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mbnsFONo3IG7",
        "outputId": "da65407d-0d5c-4252-8636-6231f222667d"
      },
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.bias', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.weight']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#model summary\n",
        "Bert.config"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3za-upXt85q8",
        "outputId": "c1fac9d3-a76c-47de-c5c7-737f9edc8abe"
      },
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BertConfig {\n",
              "  \"_name_or_path\": \"bert-base-uncased\",\n",
              "  \"architectures\": [\n",
              "    \"BertForMaskedLM\"\n",
              "  ],\n",
              "  \"attention_probs_dropout_prob\": 0.1,\n",
              "  \"classifier_dropout\": null,\n",
              "  \"gradient_checkpointing\": false,\n",
              "  \"hidden_act\": \"gelu\",\n",
              "  \"hidden_dropout_prob\": 0.1,\n",
              "  \"hidden_size\": 768,\n",
              "  \"id2label\": {\n",
              "    \"0\": \"LABEL_0\",\n",
              "    \"1\": \"LABEL_1\",\n",
              "    \"2\": \"LABEL_2\",\n",
              "    \"3\": \"LABEL_3\",\n",
              "    \"4\": \"LABEL_4\",\n",
              "    \"5\": \"LABEL_5\"\n",
              "  },\n",
              "  \"initializer_range\": 0.02,\n",
              "  \"intermediate_size\": 3072,\n",
              "  \"label2id\": {\n",
              "    \"LABEL_0\": 0,\n",
              "    \"LABEL_1\": 1,\n",
              "    \"LABEL_2\": 2,\n",
              "    \"LABEL_3\": 3,\n",
              "    \"LABEL_4\": 4,\n",
              "    \"LABEL_5\": 5\n",
              "  },\n",
              "  \"layer_norm_eps\": 1e-12,\n",
              "  \"max_position_embeddings\": 512,\n",
              "  \"model_type\": \"bert\",\n",
              "  \"num_attention_heads\": 12,\n",
              "  \"num_hidden_layers\": 12,\n",
              "  \"pad_token_id\": 0,\n",
              "  \"position_embedding_type\": \"absolute\",\n",
              "  \"transformers_version\": \"4.23.1\",\n",
              "  \"type_vocab_size\": 2,\n",
              "  \"use_cache\": true,\n",
              "  \"vocab_size\": 30522\n",
              "}"
            ]
          },
          "metadata": {},
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Create dataloader: provides a way to create a dataset out of the data,accepting data in numpy array and tensor form.\n",
        "from torch.utils.data import TensorDataset\n",
        "\n",
        "dataset_train = TensorDataset(input_ids_train, \n",
        "                              attention_masks_train,\n",
        "                              labels_train)\n",
        "dataset_val = TensorDataset(input_ids_val, \n",
        "                             attention_masks_val, \n",
        "                             labels_val)"
      ],
      "metadata": {
        "id": "5eHRsYmZ7jq0"
      },
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#To speed up the training process, make an iterator over datasets \n",
        "#reference:https://pytorch.org/docs/stable/data.html\n",
        "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
        "dataloader_train = DataLoader(dataset_train, #specify the dataset\n",
        "                              sampler = RandomSampler(dataset_train), \n",
        "                              batch_size = 32) #since we have limited resource for computation\n",
        "dataloader_val = DataLoader(dataset_val,\n",
        "                            sampler = RandomSampler(dataset_val), #returns random indices\n",
        "                            batch_size = 32) \n"
      ],
      "metadata": {
        "id": "A2ltAS-_911o"
      },
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Set up Optimizer and Scheduler"
      ],
      "metadata": {
        "id": "DV4YlWsRD_DY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#AdamW optimizer, references:https://huggingface.co/docs/transformers/v4.23.1/en/main_classes/optimizer_schedules#transformers.AdamW\n",
        "from transformers import AdamW, get_linear_schedule_with_warmup\n",
        "Optimizer=AdamW(Bert.parameters(),\n",
        "                lr=1e-5,       #The learning rate to use\n",
        "                eps=1e-8)      #Adam’s epsilon for numerical stability.default is 1e-6\n"
      ],
      "metadata": {
        "id": "Au71LHSNFZDC",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a316eafd-1318-45fe-b916-f36bd1955162"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#reference:https://huggingface.co/docs/transformers/v4.23.1/en/main_classes/optimizer_schedules#transformers.get_linear_schedule_with_warmup\n",
        "scheduler =get_linear_schedule_with_warmup(Optimizer,  #scheduler:gets called every time a batch is fed to the model.\n",
        "                                          num_warmup_steps=0, #The number of steps for the warmup phase.\n",
        "                                          num_training_steps=len(dataloader_train)*5) #set epochs=5,The total number of training steps."
      ],
      "metadata": {
        "id": "nfBBMf0sGj1Q"
      },
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "- Create a schedule with a learning rate that decreases linearly from the initial lr set in the optimizer to 0.\n",
        "\n",
        "- After a warmup period during which it increases linearly from 0 to the initial lr set in the optimizer."
      ],
      "metadata": {
        "id": "g9Xk2CfEHCw6"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Define Performance Metrics\n",
        "- Define the evaluation function\n",
        "- Define f1-score\n",
        "- Define accuracy_per_category function"
      ],
      "metadata": {
        "id": "IefIL1_OHON2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "from sklearn.metrics import f1_score"
      ],
      "metadata": {
        "id": "bYib8Vg_HRsZ"
      },
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def f1_score_func(preds, labels):\n",
        "    preds_flat = np.argmax(preds, axis = 1).flatten()  #argmax:applied over the features, returns indices of predictions (a number between 0-5) which gives classes.\n",
        "    labels_flat = labels.flatten() #flattens prediction by reshaping it into a one-dimensional tensor.\n",
        "    return f1_score(labels_flat, preds, average = 'weighted')"
      ],
      "metadata": {
        "id": "cnCkwgS-MS39"
      },
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def accuracy_per_category(preds, labels):\n",
        "    label_dict_inverse = {v: k for k, v in labelDict.items()}\n",
        "    \n",
        "    #make prediction\n",
        "    preds_flat = np.argmax(preds, axis=1).flatten()\n",
        "    labels_flat = labels.flatten()\n",
        "    \n",
        "    for label in np.unique(labels_flat):\n",
        "        y_preds = preds_flat[labels_flat==label]\n",
        "        y_true = labels_flat[labels_flat==label]\n",
        "        print(f'Class: {label_dict_inverse[label]}')\n",
        "        print(f'Accuracy:{len(y_preds[y_preds==label])}/{len(y_true)}\\n')\n"
      ],
      "metadata": {
        "id": "GwMA-4rWNjYs"
      },
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#define the device\n",
        "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
        "#As deep learning can be accellerated a lot using a GPU instead of a CPU,\n",
        "#make sure you can run this notebook in a GPU runtime (which Google Colab provides for free! - check \"Runtime\" - \"Change runtime type\" - and set the hardware accelerator to \"GPU\").\n",
        "def evaluate(dataloader_val):\n",
        "\n",
        "    #evaluation model put it in validation mode by eval() and deactivates Dropout layer\n",
        "    Bert.eval()\n",
        "    #tracking variables\n",
        "    loss_val_total = 0\n",
        "    predictions, true_vals = [], []\n",
        "    \n",
        "    for batch in tqdm(dataloader_val):\n",
        "        #unpacks DataLoader into GPU.\n",
        "        batch = tuple(b.to(device) for b in batch) #https://pytorch.org/docs/stable/generated/torch.Tensor.to.html#torch.Tensor.to\n",
        "        #define inputs\n",
        "        inputs = {'input_ids':      batch[0],\n",
        "                  'attention_mask': batch[1],\n",
        "                  'labels':         batch[2]}\n",
        "        #compute logits\n",
        "        with torch.no_grad():        \n",
        "            outputs = Bert(**inputs)\n",
        "        #compute loss\n",
        "        loss = outputs[0]\n",
        "        logits = outputs[1]\n",
        "        loss_val_total += loss.item()\n",
        "        #compute accuracy\n",
        "        logits = logits.detach().cpu().numpy()\n",
        "        label_ids = inputs['labels'].cpu().numpy()\n",
        "        predictions.append(logits)\n",
        "        true_vals.append(label_ids)\n",
        "    #compute average loss\n",
        "    loss_val_avg = loss_val_total/len(dataloader_val) \n",
        "    \n",
        "    predictions = np.concatenate(predictions, axis=0)\n",
        "    true_vals = np.concatenate(true_vals, axis=0)\n",
        "            \n",
        "    return loss_val_avg, predictions, true_vals"
      ],
      "metadata": {
        "id": "JJjEMzaaOi9t"
      },
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Train Model"
      ],
      "metadata": {
        "id": "ucEgy00yOJgV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "seed_val = 17\n",
        "random.seed(seed_val)\n",
        "np.random.seed(seed_val)\n",
        "torch.manual_seed(seed_val)\n",
        "torch.cuda.manual_seed_all(seed_val)"
      ],
      "metadata": {
        "id": "rUjPRQYVOCi5"
      },
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "epochs=5\n",
        "for epoch in tqdm(range(1, epochs+1)): \n",
        "    #train the Bert model\n",
        "    Bert.train()\n",
        "    loss_train_total = 0\n",
        "    progress_bar = tqdm(dataloader_train,    #use tqdm to track progress\n",
        "                        desc = 'Epoch {:1d}'.format(epoch),  #descriptive words\n",
        "                        leave = False,       #value of bool, whether keep the progress bar or not\n",
        "                        disable = False)\n",
        "    for batch in progress_bar:\n",
        "        Bert.zero_grad() #set gradient to 0\n",
        "        batch = tuple(b.to(device) for b in batch) #to(device):unpacks data from the DataLoader and load the data to GPU.\n",
        "        inputs = {'input_ids': batch[0],         #the inputs are divided into three parts\n",
        "                  'attention_mask': batch[1], \n",
        "                  'labels': batch[2]}\n",
        "        outputs = Bert(**inputs) #unpack the dict straight into inputs\n",
        "        \n",
        "        loss = outputs[0]\n",
        "        loss_train_total += loss.item()\n",
        "        loss.backward() #performs backward pass to compute gradients.\n",
        "        \n",
        "        torch.nn.utils.clip_grad_norm_(Bert.parameters(), 1.0) #clips the norm of the gradients to 1.0 to prevent exploding gradients.\n",
        "        \n",
        "        Optimizer.step() #updates the model’s parameters.\n",
        "        scheduler.step() #updates learning rate.\n",
        "        progress_bar.set_postfix({'training_loss': '{:.3f}'.format(loss.item()/len(batch))})\n",
        "        \n",
        "    torch.save(Bert.state_dict(), f'BERT_ft_epoch{epoch}.Bert')\n",
        "    tqdm.write('\\n Epoch {epoch}')\n",
        "    \n",
        "    loss_train_avg = loss_train_total / len(dataloader)\n",
        "    tqdm.write('Training loss: {loss_train_avg}')\n",
        "    \n",
        "    val_loss, predictions, true_vals = evaluate(dataloader_val)\n",
        "    val_f1 = f1_score_func(predictions, true_vals)\n",
        "    tqdm.write(f'Validation loss: {val_loss}')\n",
        "    tqdm.write(f'F1 Score (weighted): {val_f1}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81,
          "referenced_widgets": [
            "378712fbb3974ee9ac254d3ba1945543",
            "567c2c3dcf4e48adbeefa203a2e43784",
            "38040dc8dbfe452b8cbebdef03988d98",
            "8d0e6c0abef54d2d8ce5dc1489e6f654",
            "f932fc82aa224e6b8d96b1b97f8d08c8",
            "cd4316c436b44b5eb521162abde5dd90",
            "536773c0233b45c08d62e65200d10aad",
            "0d6a5ffbd04c42c69bfcadcf688df16e",
            "0becc2bb02504efabcaa5e4bad8f9531",
            "4ec4a13536a2405ab363c39a2a7e73dc",
            "f15ac715982c4881a5358b094ec5db97",
            "076bae934ad44a1d8c1a94e38cc34909",
            "dbd355316b4344fda71d3f6450e0da27",
            "9f219ce1fd0349e3889dda16de53f956",
            "8706354136464d1b9d3d60642a09aae9",
            "ad91880fb0934d04a9eae7ae68165fd0",
            "dc352a2841dc42e5883b9363a2dafa71",
            "2d85b26ebfbf4ce7aeaf6ebe0cc43ccd",
            "83b6d1c3db0f42f9be70d8dd8b2b3a55",
            "75e2004525684177b88362a4a18ffb1f",
            "5c1e3acb73024319bc1faa4860a634e0",
            "ed3be13cbb7f417aa79dbf3d6dcb3053"
          ]
        },
        "id": "2BHfFUoNQMGA",
        "outputId": "71dd7482-7704-4589-d804-7c7bddd09af2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/5 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "378712fbb3974ee9ac254d3ba1945543"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Epoch 1:   0%|          | 0/40 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "076bae934ad44a1d8c1a94e38cc34909"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Model Evaluation"
      ],
      "metadata": {
        "id": "aAVWD1qlvgR9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#save model\n",
        "Bert.to(device)\n",
        "pass"
      ],
      "metadata": {
        "id": "DJSh9YNn0gyZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#use dataloader_val to evaluate data\n",
        "_,predictions,true_vals=evaluate(dataloader_val)"
      ],
      "metadata": {
        "id": "mFG_oGnjzJcr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#checkout predicted labels and accuracy score\n",
        "accuracy_per_category(predictions, true_vals)"
      ],
      "metadata": {
        "id": "mOFD-HaU0Gys"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}