BERT_Embedding.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BERT Embedding",
      "provenance": [],
      "authorship_tag": "ABX9TyPCH85W5oeK5Xk2PcJKwoWn",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "e731157039e84dfcb9cc91ac80d98baf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_eb32dfe8c4d1499683270c473978d431",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_402a1d0dbba344f99ea3ff57c3770ea3",
              "IPY_MODEL_a214d3a588474773b50453e8ed15a228"
            ]
          }
        },
        "eb32dfe8c4d1499683270c473978d431": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "402a1d0dbba344f99ea3ff57c3770ea3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_99fe7a6aa2d94d0b8ed815db76b63eef",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9ddfb4fcb4d64b7784a9e0f9a291f6e2"
          }
        },
        "a214d3a588474773b50453e8ed15a228": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7983a1fa9bed4861b2641e96d44b9404",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:00&lt;00:00, 1.76MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_66b437073165463ba9980bfddf63ff78"
          }
        },
        "99fe7a6aa2d94d0b8ed815db76b63eef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9ddfb4fcb4d64b7784a9e0f9a291f6e2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7983a1fa9bed4861b2641e96d44b9404": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "66b437073165463ba9980bfddf63ff78": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "540ae0ebaa0243eabc5cf8d3ab3472cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a90377f5b5c24ee7a70105d9eca581dd",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_627398776dfe4f5189408a1e58e9d8a3",
              "IPY_MODEL_92a1714bf7e045a098a00d47d391843a"
            ]
          }
        },
        "a90377f5b5c24ee7a70105d9eca581dd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "627398776dfe4f5189408a1e58e9d8a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_825e7cc54441438ba9832881deba2f45",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 433,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 433,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a2ba11047039434294d9cb3c42b65bb9"
          }
        },
        "92a1714bf7e045a098a00d47d391843a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_129ce18b60a04a49a366f619b0614dca",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 433/433 [00:00&lt;00:00, 2.73kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0e0e1b3216db4955af8399dad78973f9"
          }
        },
        "825e7cc54441438ba9832881deba2f45": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a2ba11047039434294d9cb3c42b65bb9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "129ce18b60a04a49a366f619b0614dca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0e0e1b3216db4955af8399dad78973f9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "647309edefbc41088add8f2e00bf27f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_954932b28eac4cb1b4d6cbbfb5737b71",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_81a3c42f85a54825bc79163811676d91",
              "IPY_MODEL_ec69f724d65343f6959f4a4a949a952d"
            ]
          }
        },
        "954932b28eac4cb1b4d6cbbfb5737b71": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "81a3c42f85a54825bc79163811676d91": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_bfcc54b184db44d9a85de1332c4366af",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 440473133,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 440473133,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_87d3912522c44cd1a5aa2d77564434f2"
          }
        },
        "ec69f724d65343f6959f4a4a949a952d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a239697d17af409fa29c59c3dce8abc1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 440M/440M [00:10&lt;00:00, 40.6MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3a9d64be744d4167b1a40826dcdc2f85"
          }
        },
        "bfcc54b184db44d9a85de1332c4366af": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "87d3912522c44cd1a5aa2d77564434f2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a239697d17af409fa29c59c3dce8abc1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3a9d64be744d4167b1a40826dcdc2f85": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ludwigwittgenstein2/Research/blob/master/BERT_Embedding.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jFG6YPME6YBu"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o7buXhKk9B2o"
      },
      "source": [
        "## BERT Word Embeddings\n",
        "\n",
        "BERT (Bidirectional Encoder Representations from Transformers)\n",
        "\n",
        "BERT is a method of pretraining language representations that is used to create models that NLP practicioners\n",
        "\n",
        "We can use tasks such as classification, entity recognition, question answering\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EF3tQB7i9Zao"
      },
      "source": [
        "## Why use BERT Embedding? \n",
        "\n",
        "We use BERT to extract features, namely word and sentence embedding vectors, from text data.\n",
        "\n",
        "First, these embeddings are useful for keyword/search expansion, semantic search and information retrieval.\n",
        "\n",
        "If we want to match customer questions or searches against already answered questions or well documented searches, these representations will help us accuratley retrieve results matching the customer’s intent and contextual meaning, even if there’s no keyword or phrase overlap.\n",
        "\n",
        "BERT offers an advantage over models like Word2Vec, because while each word has a fixed representation under Word2Vec regardless of the context within which the word appears, BERT produces word representations that are dynamically informed by the words around them.\n",
        "\n",
        "“The man was accused of robbing a bank.” “The man went fishing by the bank of the river.”\n",
        "\n",
        "Word2Vec would produce the same word embedding for the word “bank” in both sentences, while under BERT the word embedding for “bank” would be different for each sentence\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4iMt4skLAKGf"
      },
      "source": [
        "## Loading Pre-Trained BERT"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wU9AgUtw63Jy",
        "outputId": "64a5f3c6-2ed9-44cc-b82f-c580c6f7f197"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f9/54/5ca07ec9569d2f232f3166de5457b63943882f7950ddfcc887732fc7fb23/transformers-4.3.3-py3-none-any.whl (1.9MB)\n",
            "\u001b[K     |████████████████████████████████| 1.9MB 4.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 23.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (3.4.0)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/71/23/2ddc317b2121117bf34dd00f5b0de194158f2a44ee2bf5e47c7166878a97/tokenizers-0.10.1-cp37-cp37m-manylinux2010_x86_64.whl (3.2MB)\n",
            "\u001b[K     |████████████████████████████████| 3.2MB 36.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp37-none-any.whl size=893262 sha256=fde7c58f50528a97b3edfc9c9dc97e4bc74466810416a4d9c906baf4c1f6eedc\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: sacremoses, tokenizers, transformers\n",
            "Successfully installed sacremoses-0.0.43 tokenizers-0.10.1 transformers-4.3.3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "96O3BHmjA0aR"
      },
      "source": [
        "## transformers\n",
        "\n",
        "provides a number of classes for applying BERT to different tasks (token classification, text classification, …). Here, we’re using the basic BertModel which has no specific output task–it’s a good choice for using BERT just to extract embeddings."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jOvkbd_iBP9i"
      },
      "source": [
        "## Input Formatting\n",
        "\n",
        "A special token, [SEP], to mark the end of a sentence, or the separation between two sentences\n",
        "\n",
        "A special token, [CLS], at the beginning of our text. \n",
        "\n",
        "This token is used for classification tasks, but BERT expects it no matter what your application is. Tokens that conform with the fixed vocabulary used in BERT\n",
        "\n",
        "The Token IDs for the tokens, from BERT’s tokenizer\n",
        "\n",
        "Mask IDs to indicate which elements in the sequence are tokens and which are padding elements\n",
        "\n",
        "Segment IDs used to distinguish different sentences\n",
        "\n",
        "Positional Embeddings used to show token position within the sequence"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3OnAQBoSBrSt"
      },
      "source": [
        "#Special Tokens\n",
        "\n",
        "BERT can take as input either one or two sentences, and uses the special token [SEP] to differentiate them. The [CLS] token always appears at the start of the text, and is specific to classification tasks.\n",
        "\n",
        "\n",
        "##2 Sentence Input:\n",
        "\n",
        "[CLS] The man went to the store. [SEP] He bought a gallon of milk.\n",
        "\n",
        "## 1 Sentence Input:\n",
        "\n",
        "[CLS] The man went to the store. [SEP]\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "e731157039e84dfcb9cc91ac80d98baf",
            "eb32dfe8c4d1499683270c473978d431",
            "402a1d0dbba344f99ea3ff57c3770ea3",
            "a214d3a588474773b50453e8ed15a228",
            "99fe7a6aa2d94d0b8ed815db76b63eef",
            "9ddfb4fcb4d64b7784a9e0f9a291f6e2",
            "7983a1fa9bed4861b2641e96d44b9404",
            "66b437073165463ba9980bfddf63ff78"
          ]
        },
        "id": "sXLaDhWL64en",
        "outputId": "0fe05ec6-bc5a-409b-9ada-33af858b6993"
      },
      "source": [
        "import torch\n",
        "from transformers import BertTokenizer, BertModel\n",
        "\n",
        "# OPTIONAL: if you want to have more information on what's happening, activate the logger as follows\n",
        "import logging\n",
        "#logging.basicConfig(level=logging.INFO)\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "% matplotlib inline\n",
        "\n",
        "# Load pre-trained model tokenizer (vocabulary)\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e731157039e84dfcb9cc91ac80d98baf",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VuygrzTBCKZ7"
      },
      "source": [
        "\n",
        "## Tokenization\n",
        "\n",
        "BERT provides its own tokenizer, which we imported above. We can see how it handles the sentence below\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TQrOUxe669Yv",
        "outputId": "1021a084-f460-4b74-9b3f-2c026cc7a65c"
      },
      "source": [
        "text = \"Here is the sentence I want embeddings for.\"\n",
        "marked_text = \"[CLS] \" + text + \" [SEP]\"\n",
        "\n",
        "# Tokenize our sentence with the BERT tokenizer.\n",
        "tokenized_text = tokenizer.tokenize(marked_text)\n",
        "\n",
        "# Print out the tokens.\n",
        "print (tokenized_text)\n",
        "\n",
        "# We can see how the word “embeddings” is represented: \n",
        "# The original word has been split into smaller subwords and characters.\n"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['[CLS]', 'here', 'is', 'the', 'sentence', 'i', 'want', 'em', '##bed', '##ding', '##s', 'for', '.', '[SEP]']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E0fxluvVC3zl"
      },
      "source": [
        "##Why does it look this way? \n",
        "\n",
        "This is because the BERT tokenizer was created with a WordPiece model. This model greedily creates a fixed-size vocabulary of individual characters, subwords, and words that best fits our language data. \n",
        "\n",
        "Since the vocabulary limit size of our BERT tokenizer model is 30,000, the WordPiece model generated a vocabulary that contains all English characters plus the ~30,000 most common words and subwords found in the English language corpus the model is trained on.\n",
        "\n",
        "1)Whole words\n",
        "\n",
        "2)Subwords occuring at the front of a word or in isolation (“em” as in “embeddings” is assigned the same vector as the standalone sequence of characters “em” as in “go get em” )\n",
        "\n",
        "3)Subwords not at the front of a word, which are preceded by ‘##’ to denote this case\n",
        "\n",
        "4)Individual characters\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lOJz3X5c7FQd",
        "outputId": "77d5a3f6-34fa-428d-e946-52710f7b883a"
      },
      "source": [
        "# Define a new example sentence with multiple meanings of the word \"bank\"\n",
        "text = \"After stealing money from the bank vault, the bank robber was seen \" \\\n",
        "       \"fishing on the Mississippi river bank.\"\n",
        "\n",
        "# Add the special tokens.\n",
        "marked_text = \"[CLS] \" + text + \" [SEP]\"\n",
        "\n",
        "# Split the sentence into tokens.\n",
        "tokenized_text = tokenizer.tokenize(marked_text)\n",
        "\n",
        "# Map the token strings to their vocabulary indeces.\n",
        "indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)\n",
        "\n",
        "# Display the words with their indeces.\n",
        "for tup in zip(tokenized_text, indexed_tokens):\n",
        "    print('{:<12} {:>6,}'.format(tup[0], tup[1]))"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[CLS]           101\n",
            "after         2,044\n",
            "stealing     11,065\n",
            "money         2,769\n",
            "from          2,013\n",
            "the           1,996\n",
            "bank          2,924\n",
            "vault        11,632\n",
            ",             1,010\n",
            "the           1,996\n",
            "bank          2,924\n",
            "robber       27,307\n",
            "was           2,001\n",
            "seen          2,464\n",
            "fishing       5,645\n",
            "on            2,006\n",
            "the           1,996\n",
            "mississippi   5,900\n",
            "river         2,314\n",
            "bank          2,924\n",
            ".             1,012\n",
            "[SEP]           102\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wFdW8uYZERoC"
      },
      "source": [
        "#Segment ID\n",
        "\n",
        "BERT is trained on and expects sentence pairs, using 1s and 0s to distinguish between the two sentences. \n",
        "\n",
        "To process two sentences, assign each word in the first sentence plus the ‘[SEP]’ token a 0, and all tokens of the second sentence a 1."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FqgA1b6D7ICU",
        "outputId": "4140acbf-a203-4e61-c10b-3aaad139b0fa"
      },
      "source": [
        "# Mark each of the 22 tokens as belonging to sentence \"1\".\n",
        "segments_ids = [1] * len(tokenized_text)\n",
        "\n",
        "print (segments_ids)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sC-bFXaF7LS1"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LETJj4qTEoAh"
      },
      "source": [
        "#Extracting Embeddings\n",
        "\n",
        "##Running BERT on our text\n",
        "\n",
        "We need to convert our data to torch tensors and call the BERT model\n",
        "\n",
        "Next, let’s evaluate BERT on our example text"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EYW6EKC_7LxZ"
      },
      "source": [
        ""
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AhrV4c1b7QHf"
      },
      "source": [
        "# Convert inputs to PyTorch tensors\n",
        "tokens_tensor = torch.tensor([indexed_tokens])\n",
        "segments_tensors = torch.tensor([segments_ids])"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CzZx3NpwFQ7R"
      },
      "source": [
        "#the Output\n",
        "\n",
        "1.The layer number (13 layers)\n",
        "\n",
        "2.The batch number (1 sentence)\n",
        "\n",
        "3.The word / token number (22 tokens in our sentence)\n",
        "\n",
        "4.The hidden unit / feature number (768 features)\n",
        "\n",
        "Wait, 13 layers? Doesn’t BERT only have 12? It’s 13 because the first element is the input embeddings, the rest is the outputs of each of BERT’s 12 layers.\n",
        "\n",
        "That’s 219,648 unique values just to represent our one sentence!\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "540ae0ebaa0243eabc5cf8d3ab3472cb",
            "a90377f5b5c24ee7a70105d9eca581dd",
            "627398776dfe4f5189408a1e58e9d8a3",
            "92a1714bf7e045a098a00d47d391843a",
            "825e7cc54441438ba9832881deba2f45",
            "a2ba11047039434294d9cb3c42b65bb9",
            "129ce18b60a04a49a366f619b0614dca",
            "0e0e1b3216db4955af8399dad78973f9",
            "647309edefbc41088add8f2e00bf27f5",
            "954932b28eac4cb1b4d6cbbfb5737b71",
            "81a3c42f85a54825bc79163811676d91",
            "ec69f724d65343f6959f4a4a949a952d",
            "bfcc54b184db44d9a85de1332c4366af",
            "87d3912522c44cd1a5aa2d77564434f2",
            "a239697d17af409fa29c59c3dce8abc1",
            "3a9d64be744d4167b1a40826dcdc2f85"
          ]
        },
        "id": "fmygPNF27Bz7",
        "outputId": "444a51de-1d93-49f2-ffc8-1d3df80d23d0"
      },
      "source": [
        "# Load pre-trained model (weights)\n",
        "model = BertModel.from_pretrained('bert-base-uncased',\n",
        "                                  output_hidden_states = True, # Whether the model returns all hidden-states.\n",
        "                                  )\n",
        "\n",
        "# Put the model in \"evaluation\" mode, meaning feed-forward operation.\n",
        "model.eval()"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "540ae0ebaa0243eabc5cf8d3ab3472cb",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=433.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "647309edefbc41088add8f2e00bf27f5",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=440473133.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BertModel(\n",
              "  (embeddings): BertEmbeddings(\n",
              "    (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
              "    (position_embeddings): Embedding(512, 768)\n",
              "    (token_type_embeddings): Embedding(2, 768)\n",
              "    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "    (dropout): Dropout(p=0.1, inplace=False)\n",
              "  )\n",
              "  (encoder): BertEncoder(\n",
              "    (layer): ModuleList(\n",
              "      (0): BertLayer(\n",
              "        (attention): BertAttention(\n",
              "          (self): BertSelfAttention(\n",
              "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (output): BertSelfOutput(\n",
              "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (intermediate): BertIntermediate(\n",
              "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "        )\n",
              "        (output): BertOutput(\n",
              "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (1): BertLayer(\n",
              "        (attention): BertAttention(\n",
              "          (self): BertSelfAttention(\n",
              "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (output): BertSelfOutput(\n",
              "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (intermediate): BertIntermediate(\n",
              "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "        )\n",
              "        (output): BertOutput(\n",
              "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (2): BertLayer(\n",
              "        (attention): BertAttention(\n",
              "          (self): BertSelfAttention(\n",
              "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (output): BertSelfOutput(\n",
              "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (intermediate): BertIntermediate(\n",
              "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "        )\n",
              "        (output): BertOutput(\n",
              "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (3): BertLayer(\n",
              "        (attention): BertAttention(\n",
              "          (self): BertSelfAttention(\n",
              "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (output): BertSelfOutput(\n",
              "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (intermediate): BertIntermediate(\n",
              "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "        )\n",
              "        (output): BertOutput(\n",
              "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (4): BertLayer(\n",
              "        (attention): BertAttention(\n",
              "          (self): BertSelfAttention(\n",
              "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (output): BertSelfOutput(\n",
              "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (intermediate): BertIntermediate(\n",
              "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "        )\n",
              "        (output): BertOutput(\n",
              "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (5): BertLayer(\n",
              "        (attention): BertAttention(\n",
              "          (self): BertSelfAttention(\n",
              "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (output): BertSelfOutput(\n",
              "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (intermediate): BertIntermediate(\n",
              "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "        )\n",
              "        (output): BertOutput(\n",
              "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (6): BertLayer(\n",
              "        (attention): BertAttention(\n",
              "          (self): BertSelfAttention(\n",
              "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (output): BertSelfOutput(\n",
              "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (intermediate): BertIntermediate(\n",
              "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "        )\n",
              "        (output): BertOutput(\n",
              "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (7): BertLayer(\n",
              "        (attention): BertAttention(\n",
              "          (self): BertSelfAttention(\n",
              "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (output): BertSelfOutput(\n",
              "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (intermediate): BertIntermediate(\n",
              "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "        )\n",
              "        (output): BertOutput(\n",
              "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (8): BertLayer(\n",
              "        (attention): BertAttention(\n",
              "          (self): BertSelfAttention(\n",
              "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (output): BertSelfOutput(\n",
              "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (intermediate): BertIntermediate(\n",
              "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "        )\n",
              "        (output): BertOutput(\n",
              "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (9): BertLayer(\n",
              "        (attention): BertAttention(\n",
              "          (self): BertSelfAttention(\n",
              "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (output): BertSelfOutput(\n",
              "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (intermediate): BertIntermediate(\n",
              "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "        )\n",
              "        (output): BertOutput(\n",
              "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (10): BertLayer(\n",
              "        (attention): BertAttention(\n",
              "          (self): BertSelfAttention(\n",
              "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (output): BertSelfOutput(\n",
              "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (intermediate): BertIntermediate(\n",
              "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "        )\n",
              "        (output): BertOutput(\n",
              "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (11): BertLayer(\n",
              "        (attention): BertAttention(\n",
              "          (self): BertSelfAttention(\n",
              "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (output): BertSelfOutput(\n",
              "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (intermediate): BertIntermediate(\n",
              "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "        )\n",
              "        (output): BertOutput(\n",
              "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (pooler): BertPooler(\n",
              "    (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "    (activation): Tanh()\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E3QccAaRGCgu"
      },
      "source": [
        "Grouping the values by layer makes sense for the model, but for our purposes we want it grouped by token.\n",
        "\n",
        "Current dimensions:\n",
        "\n",
        "[# layers, # batches, # tokens, # features]\n",
        "\n",
        "Desired dimensions:\n",
        "\n",
        "[# tokens, # layers, # features]\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XquK-GUG7mqC"
      },
      "source": [
        "# Run the text through BERT, and collect all of the hidden states produced\n",
        "# from all 12 layers. \n",
        "with torch.no_grad():\n",
        "\n",
        "    outputs = model(tokens_tensor, segments_tensors)\n",
        "\n",
        "    # Evaluating the model will return a different number of objects based on \n",
        "    # how it's  configured in the `from_pretrained` call earlier. In this case, \n",
        "    # becase we set `output_hidden_states = True`, the third item will be the \n",
        "    # hidden states from all layers. See the documentation for more details:\n",
        "    # https://huggingface.co/transformers/model_doc/bert.html#bertmodel\n",
        "    hidden_states = outputs[2]"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3Y5MaHJG78lY",
        "outputId": "e133c58a-aa58-439f-c919-1e3ad70c4a95"
      },
      "source": [
        "print (\"Number of layers:\", len(hidden_states), \"  (initial embeddings + 12 BERT layers)\")\n",
        "layer_i = 0\n",
        "\n",
        "print (\"Number of batches:\", len(hidden_states[layer_i]))\n",
        "batch_i = 0\n",
        "\n",
        "print (\"Number of tokens:\", len(hidden_states[layer_i][batch_i]))\n",
        "token_i = 0\n",
        "\n",
        "print (\"Number of hidden units:\", len(hidden_states[layer_i][batch_i][token_i]))"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of layers: 13   (initial embeddings + 12 BERT layers)\n",
            "Number of batches: 1\n",
            "Number of tokens: 22\n",
            "Number of hidden units: 768\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 592
        },
        "id": "n38FPawi7_oa",
        "outputId": "c7b4c5e1-95c1-4a10-b435-040a138f797d"
      },
      "source": [
        "# For the 5th token in our sentence, select its feature values from layer 5.\n",
        "token_i = 5\n",
        "layer_i = 5\n",
        "vec = hidden_states[layer_i][batch_i][token_i]\n",
        "\n",
        "# Plot the values as a histogram to show their distribution.\n",
        "plt.figure(figsize=(10,10))\n",
        "plt.hist(vec, bins=200)\n",
        "plt.show()"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlAAAAI/CAYAAAC4QOfKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAVxElEQVR4nO3df4xl93nX8c+DJwkVBdLgqbHimHFVl8qhxEFbK6hFKHaTurjUBtooFYJFWFpRCkogqEwShFQJpE2LmlYI/rDqqAtKSdImwVanQI2bUkDE6To/mjhuiBs2NI4Tb0qipkKkcvPwx1yna++uZ56dH/fuzuslWXPPuXfmPv5qd+a95945p7o7AADs3h9a9gAAAJcbAQUAMCSgAACGBBQAwJCAAgAYElAAAENrh/lkV199dW9sbBzmUwIAXJKHH374C929fqH7DjWgNjY2cvr06cN8SgCAS1JVn77YfV7CAwAYElAAAEMCCgBgSEABAAwJKACAIQEFADAkoAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwJKAAAIYEFADAkIACABgSUAAAQwIKAGBIQAEADAkoAIAhAQUAMCSgAACGBBQAwJCAAoDL3MbmVjY2t5Y9xpEioAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwJKAAAIYEFADAkIACABgSUAAAQwIKAGBIQAEADAkoAIAhAQUAMCSgAACGBBQAwJCAAgAYElAAAEMCCgBgSEABAAwJKACAIQEFADAkoAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwJKAAAIYEFADAkIACABhaW/YAAMD+2Njc+trtMyfvWOIkVz5HoAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwJKAAAIYEFADAkIACABgSUAAAQwIKAGBIQAEADAkoAIAhAQUAMCSgAACGBBQAwJCAAgAYElAAAEMCCgBgSEABAAwJKACAobXdPKiqziT5cpLfT/JUdx+rqhcleWeSjSRnkrymu794MGMCAKyOyRGoV3b3zd19bLG9meTB7r4xyYOLbQCAK95eXsK7M8mpxe1TSe7a+zgAAKtvtwHVSX6pqh6uqhOLfdd09xOL259Lcs2+TwcAsIJ29R6oJN/Z3Y9X1TcmeaCqfuPcO7u7q6ov9ImL4DqRJNdff/2ehgUAWAW7OgLV3Y8vPj6Z5L1Jbkny+aq6NkkWH5+8yOfe093HuvvY+vr6/kwNALBEOwZUVf2RqvqjT99O8uokH0tyf5Lji4cdT3LfQQ0JALBKdvMS3jVJ3ltVTz/+Z7v7P1bVryV5V1XdneTTSV5zcGMCAKyOHQOquz+V5GUX2P/bSW47iKEAAFaZM5EDAAwJKACAIQEFADAkoAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwJKAAAIYEFADAkIACABgSUAAAQwIKAGBIQAEADAkoAIAhAQUAMCSgAACGBBQAwJCAAgAYElAAAEMCCgBgSEABAAwJKACAIQEFADAkoAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwJKAAAIYEFADAkIACABgSUAAAQwIKAGBIQAEADAkoAIAhAQUAMCSgAACGBBQAwJCAAgAYElAAAEMCCgBgSEABAAwJKACAIQEFADAkoAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQBXoI3NrWxsbu24j0sjoAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwtLbsAQCAg+PEmQfDESgAgCEBBQAwJKAAAIYEFADAkIACABgSUAAAQwIKAGBIQAEADAkoADjCNja3nGzzEggoAIAhAQUAMCSgAACGBBQAwNCuA6qqrqqqD1XVLyy2b6iqh6rqsap6Z1U9/+DGBABYHZMjUK9L8ug5229J8tbu/uYkX0xy934OBgCwqnYVUFV1XZI7kvz0YruS3Jrk5xcPOZXkroMYEABg1ez2CNRPJvmRJF9dbP+JJF/q7qcW259J8uJ9ng0AYCXtGFBV9b1Jnuzuhy/lCarqRFWdrqrTZ8+evZQvAQAsOPHlatjNEajvSPJ9VXUmyTuy/dLdTyV5YVWtLR5zXZLHL/TJ3X1Pdx/r7mPr6+v7MDIAwHLtGFDd/cbuvq67N5K8Nskvd/dfT/K+JN+/eNjxJPcd2JQAACtkL+eB+sdJ/mFVPZbt90Tduz8jAQCstrWdH/IHuvtXkvzK4vanktyy/yMBAKw2ZyIHABgSUAAAQwIKAGBIQAEADAkoAIAhAQUAMCSgAACGBBQAwJCAAgAYElAAAEMCCgBgSEABAAwJKACAIQEFADC0tuwBAIC5jc2tZY9wpDkCBQAwJKAAAIYEFADAkIACABgSUAAAQwIKAGBIQAEADAkoAIAhJ9IEgCPmQifhPHffmZN3HOY4lyVHoAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwJKAAAIYEFADAkIACABgSUAAAQwIKAGBIQAEADAkoAIAhAQUAMCSgAACG1pY9AADw3DY2t5Y9As/iCBQAwJCAAgAYElAAAEMCCgBgSEABAAwJKACAIQEFADAkoAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwJKAAAIYEFADAkIACABgSUAAAQwIKAGBIQAEADAkoAIAhAQUAMCSgAACGBBQAwJCAAgAYElAAAEMCCgBgSEABAAwJKACAIQEFADAkoAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwJKAAAIZ2DKiq+sNV9YGq+khVPVJVP7rYf0NVPVRVj1XVO6vq+Qc/LgDA8u3mCNRXktza3S9LcnOS26vqFUnekuSt3f3NSb6Y5O6DGxMAYHXsGFC97XcXm89b/NdJbk3y84v9p5LcdSATAgCsmF29B6qqrqqqDyd5MskDSX4zyZe6+6nFQz6T5MUHMyIAwGrZVUB19+93981JrktyS5Jv3e0TVNWJqjpdVafPnj17iWMCAIdlY3MrG5tbyx5jpY1+C6+7v5TkfUn+fJIXVtXa4q7rkjx+kc+5p7uPdfex9fX1PQ0LALAKdvNbeOtV9cLF7a9L8qokj2Y7pL5/8bDjSe47qCEBAFbJ2s4PybVJTlXVVdkOrnd19y9U1ceTvKOq/lmSDyW59wDnBABYGTsGVHf/epKXX2D/p7L9figAgCPFmcgBAIYEFADAkIACABgSUAAAQwIKAGBIQAEADAkoAIAhAQUAMCSgAACGBBQAwJCAAgAYElAAAEMCCgBgSEABAAwJKACAIQEFADAkoAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwJKAAAIYEFADAkIACABgSUAAAQwIKAGBIQAEADAkoAIAhAQUAMCSgAACGBBQAwJCAAgAYElAAAEMCCgBgSEABAAwJKACAIQEFADAkoAAAhgQUAKyQjc2tbGxuLXsMdiCgAACGBBQAwJCAAgAYElAAAEMCCgBgSEABAAwJKACAIQEFADC0tuwBAIDzOZnmanMECgBgSEABAAwJKACAIQEFADAkoAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwJKAAAIYEFADAkIACABgSUAAAQwIKAGBobdkDAMBRt7G5tewRGHIECgBgSEABAAwJKACAIQEFADAkoAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwJKAAAIZ2DKiqeklVva+qPl5Vj1TV6xb7X1RVD1TVJxcfv+HgxwUAWL7dHIF6KskbuvumJK9I8sNVdVOSzSQPdveNSR5cbAMAXPF2DKjufqK7P7i4/eUkjyZ5cZI7k5xaPOxUkrsOakgAgFUyeg9UVW0keXmSh5Jc091PLO76XJJr9nUyAIAVtbbbB1bV1yd5d5LXd/fvVNXX7uvurqq+yOedSHIiSa6//vq9TQsAHJqNza2v3T5z8o4lTrJ6dnUEqqqel+14ent3v2ex+/NVde3i/muTPHmhz+3ue7r7WHcfW19f34+ZAQCWaje/hVdJ7k3yaHf/xDl33Z/k+OL28ST37f94AACrZzcv4X1Hkr+R5KNV9eHFvjclOZnkXVV1d5JPJ3nNwYwIALBadgyo7v5vSeoid9+2v+MAAKw+ZyIHABgSUAAAQwIKAGBIQAEADAkoAIAhAQUAMCSgAACGBBQAwJCAAgAYElAAAEMCCgBgSEABAAwJKACAobVlDwAArL6Nza3z9p05eccSJlkNjkABAAwJKACAIQEFADAkoAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwJKAAAIYEFADAkIACABgSUAAAQwIKAGBIQAEADAkoAIAhAQUAMCSgAACGBBQAwJCAAgAYElAAAEMCCgBgSEABAAwJKACAIQEFADAkoAAAhgQUAMCQgAIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwJKAAAIYEFADAkIACABgSUAAAQwIKAGBIQAEADAkoAIAhAQUAMCSgAACGBBQAwNDasgcAgKNqY3Nr2SPsybnznzl5xxInOXyOQAEADAkoAIAhAQUAMCSgAACGBBQAwJCAAgAYElAAAEMCCgBgyIk0AeCAXO4nypy40P/rlXxyTUegAACGBBQAwJCAAgAYElAAAEMCCgBgSEABAAwJKACAIQEFADAkoAAAhgQUAMCQgAIAGBJQAABDAgoAYGjHgKqqt1XVk1X1sXP2vaiqHqiqTy4+fsPBjgkAsDp2cwTqZ5Lc/qx9m0ke7O4bkzy42AYAOBJ2DKju/tUk/+dZu+9Mcmpx+1SSu/Z5LgCAlXWp74G6prufWNz+XJJr9mkeAICVt+c3kXd3J+mL3V9VJ6rqdFWdPnv27F6fDgBg6S41oD5fVdcmyeLjkxd7YHff093HuvvY+vr6JT4dAMDquNSAuj/J8cXt40nu259xAABW325OY/DvkvyPJH+6qj5TVXcnOZnkVVX1ySTftdgGADgS1nZ6QHf/4EXuum2fZwEAuCw4EzkAwJCAAgAYElAAAEMCCgBgaMc3kQMAMxubW8segQPmCBQAwJCAAgAYElAAAEMCCgBgSEABAAwJKACAIQEFADAkoAAAhgQUAHAgNja3rtiTigooAIAhAQUAMCSgAACGBBQAwJCAAgAYElAAAEMCCgBgSEABAAytLXsAALjcnHtyyDMn71jiJCyLI1AAAEMCCgBgSEABAAwJKACAIQEFADAkoAAAhgQUAMCQgAIAGHIiTQDgQF2JJx51BAoAYEhAAQAMCSgAgCEBBQAwJKAAAIYEFADAkIACABgSUAAAQ06kCQAcusv95JqOQAEADAkoAIAhAQUAMCSgAACGBBQAwJCAAgAYElAAAEMCCgBgSEABwB5sbG4946SQzF2OayigAACGBBQAwJCAAgAYElAAAEMCCgBgSEABAAwJKACAIQEFADAkoAAAhtaWPQAAcHRMzzh+7uPPnLxjv8e5ZI5AAQAMCSgAgCEBBQAwJKAAAIYEFADAkIACABgSUAAAQwIKAGDIiTQB4Dns9kSO0xNE8txWfT0dgQIAGBJQAABDAgoAYEhAAQAMCSgAgCEBBQAwJKAAAIYEFADA0BV3Is3dnvAMAC7mYidxXPWTO17pnl7/Vfj57ggUAMCQgAIAGBJQAABDAgoAYGhPAVVVt1fVJ6rqsara3K+hAABW2SUHVFVdleRfJfmeJDcl+cGqumm/BgMAWFV7OQJ1S5LHuvtT3f17Sd6R5M79GQsAYHXtJaBenOS3ztn+zGIfAMAV7cBPpFlVJ5KcWGz+blV94qCf82vP/ZbDeqaRq5N8YdlDrBhrcj5rcj5rcj5rcj5r8kyX1Xrs9uf2Hn++T9bkT13sjr0E1ONJXnLO9nWLfc/Q3fckuWcPz3NFqarT3X1s2XOsEmtyPmtyPmtyPmtyPmvyTNbjfPu1Jnt5Ce/XktxYVTdU1fOTvDbJ/XsdCABg1V3yEajufqqq/l6S/5TkqiRv6+5H9m0yAIAVtaf3QHX3Lyb5xX2a5ajwcub5rMn5rMn5rMn5rMn5rMkzWY/z7cuaVHfvx9cBADgyXMoFAGBIQB2SqvqBqnqkqr5aVcfO2f+qqnq4qj66+HjrMuc8TBdbk8V9b1xcIugTVfXdy5pxmarq5qp6f1V9uKpOV9Uty55p2arq71fVbyz+3PzYsudZFVX1hqrqqrp62bMsW1X9+OLPyK9X1Xur6oXLnmlZXG7tmarqJVX1vqr6+OJ7yOv28vUE1OH5WJK/muRXn7X/C0n+cnd/W5LjSf7tYQ+2RBdck8UlgV6b5KVJbk/yrxeXDjpqfizJj3b3zUn+6WL7yKqqV2b7agcv6+6XJvkXSx5pJVTVS5K8Osn/XvYsK+KBJH+mu/9skv+Z5I1LnmcpXG7tgp5K8obuvinJK5L88F7WREAdku5+tLvPO4lod3+ouz+72HwkyddV1QsOd7rluNiaZPuH5Du6+yvd/b+SPJbtSwcdNZ3kjy1u//Ekn32Oxx4FP5TkZHd/JUm6+8klz7Mq3prkR7L95+XI6+5f6u6nFpvvz/Y5Co8il1t7lu5+ors/uLj95SSPZg9XUBFQq+WvJfng0z8gjjCXCdr2+iQ/XlW/le2jLUfyX9Ln+JYkf6GqHqqq/1JV377sgZatqu5M8nh3f2TZs6yov53kPyx7iCXxffQ5VNVGkpcneehSv8aBX8rlKKmq/5zkT17grjd39307fO5Lk7wl24firxh7WZOj4LnWJ8ltSf5Bd7+7ql6T5N4k33WY8x22HdZjLcmLsn3o/duTvKuqvqmv8F8l3mFN3pQr7HvGbuzm+0pVvTnbL9m8/TBnY/VV1dcneXeS13f371zq1xFQ+6i7L+mHW1Vdl+S9Sf5md//m/k61XJe4Jru6TNCV4LnWp6r+TZKn3+T4c0l++lCGWqId1uOHkrxnEUwfqKqvZvuaVmcPa75luNiaVNW3JbkhyUeqKtn+e/LBqrqluz93iCMeup2+r1TV30ryvUluu9ID+zkcme+jE1X1vGzH09u7+z17+VpewluyxW+IbCXZ7O7/vux5VsT9SV5bVS+oqhuS3JjkA0ueaRk+m+QvLm7fmuSTS5xlFfz7JK9Mkqr6liTPz2V0kdT91t0f7e5v7O6N7t7I9ks0f+5Kj6edVNXt2X5P2Pd19/9d9jxL5HJrz1Lb/9K4N8mj3f0Te/56RzfOD1dV/ZUk/zLJepIvJflwd393Vf2TbL+35dwfjq8+Cm+QvdiaLO57c7bfv/BUtg+zHrn3MVTVdyb5qWwfKf5/Sf5udz+83KmWZ/FD4G1Jbk7ye0n+UXf/8nKnWh1VdSbJse4+slGZJFX1WJIXJPntxa73d/ffWeJIS1NVfynJT+YPLrf2z5c80lItvqf+1yQfTfLVxe43La6qMv96AgoAYMZLeAAAQwIKAGBIQAEADAkoAIAhAQUAMCSgAACGBBQAwJCAAgAY+v8yrajMccmJFwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 720x720 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vf_wdNF18DPJ",
        "outputId": "f463b03e-ca59-41cf-904d-82dca3cdddb9"
      },
      "source": [
        "# `hidden_states` is a Python list.\n",
        "print('      Type of hidden_states: ', type(hidden_states))\n",
        "\n",
        "# Each layer in the list is a torch tensor.\n",
        "print('Tensor shape for each layer: ', hidden_states[0].size())"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "      Type of hidden_states:  <class 'tuple'>\n",
            "Tensor shape for each layer:  torch.Size([1, 22, 768])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cbYKHdEbKcB6"
      },
      "source": [
        "## We combine the layers to make this one whole big tensor."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I9A4DdtL8GlL",
        "outputId": "c8c73034-cada-4d4c-b12f-7fc15f4cdcfa"
      },
      "source": [
        "# Concatenate the tensors for all layers. We use `stack` here to\n",
        "# create a new dimension in the tensor.\n",
        "token_embeddings = torch.stack(hidden_states, dim=0)\n",
        "\n",
        "token_embeddings.size()"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([13, 1, 22, 768])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8wxfrhgo8IvQ",
        "outputId": "76a09b2b-11d5-45ea-b62d-da51ad7502d7"
      },
      "source": [
        "torch.Size([13, 1, 22, 768])"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([13, 1, 22, 768])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6sGMf-vi8Kxd",
        "outputId": "434fc57b-298a-43a8-eff0-081de560da56"
      },
      "source": [
        "# Remove dimension 1, the \"batches\".\n",
        "token_embeddings = torch.squeeze(token_embeddings, dim=1)\n",
        "\n",
        "token_embeddings.size()"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([13, 22, 768])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NkR3q8tv8Lts",
        "outputId": "a45dc3f7-2cb2-4eae-9f88-88b2381d20f1"
      },
      "source": [
        "torch.Size([13, 22, 768])"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([13, 22, 768])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vHB-H2_g8N6H",
        "outputId": "d31dc6c8-17d1-44d1-e42f-905a5ed7998d"
      },
      "source": [
        "# Swap dimensions 0 and 1.\n",
        "token_embeddings = token_embeddings.permute(1,0,2)\n",
        "\n",
        "token_embeddings.size()"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([22, 13, 768])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rfo6QEqg8Qdq"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AO4OTusYLamQ"
      },
      "source": [
        "### Creating word and sentence vectors from hidden states\n",
        "\n",
        "what do we do with these hidden states? \n",
        "\n",
        "We would like to get individual vectors for each of our tokens, or perhaps a single vector representation of the whole sentence, but for each token of our input we have 13 separate vectors each of length 768.\n",
        "\n",
        "## Word Vectors\n",
        "\n",
        "Concatenate the last four layers, giving us a single word vector per token. Each vector will have length 4 x 768 = 3,072.\n",
        "\n",
        "Alternatively, we can create the word vectors by summing together the last four layers.\n",
        "\n",
        "## Sentence Vectors\n",
        "To get a single vector for our entire sentence we have multiple application-dependent strategies, but a simple approach is to average the second to last hiden layer of each token producing a single 768 length vector.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lGXJmniT8Qre",
        "outputId": "ea2632b8-996a-4448-b47d-16d32c190734"
      },
      "source": [
        "# Stores the token vectors, with shape [22 x 3,072]\n",
        "token_vecs_cat = []\n",
        "\n",
        "# `token_embeddings` is a [22 x 12 x 768] tensor.\n",
        "\n",
        "# For each token in the sentence...\n",
        "for token in token_embeddings:\n",
        "    \n",
        "    # `token` is a [12 x 768] tensor\n",
        "\n",
        "    # Concatenate the vectors (that is, append them together) from the last \n",
        "    # four layers.\n",
        "    # Each layer vector is 768 values, so `cat_vec` is length 3,072.\n",
        "    cat_vec = torch.cat((token[-1], token[-2], token[-3], token[-4]), dim=0)\n",
        "    \n",
        "    # Use `cat_vec` to represent `token`.\n",
        "    token_vecs_cat.append(cat_vec)\n",
        "\n",
        "print ('Shape is: %d x %d' % (len(token_vecs_cat), len(token_vecs_cat[0])))"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Shape is: 22 x 3072\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0GBra6Z98WH1",
        "outputId": "c12b6cd4-4f53-40cc-9d4e-4f20ce971fee"
      },
      "source": [
        "# Stores the token vectors, with shape [22 x 768]\n",
        "token_vecs_sum = []\n",
        "\n",
        "# `token_embeddings` is a [22 x 12 x 768] tensor.\n",
        "\n",
        "# For each token in the sentence...\n",
        "for token in token_embeddings:\n",
        "\n",
        "    # `token` is a [12 x 768] tensor\n",
        "\n",
        "    # Sum the vectors from the last four layers.\n",
        "    sum_vec = torch.sum(token[-4:], dim=0)\n",
        "    \n",
        "    # Use `sum_vec` to represent `token`.\n",
        "    token_vecs_sum.append(sum_vec)\n",
        "\n",
        "print ('Shape is: %d x %d' % (len(token_vecs_sum), len(token_vecs_sum[0])))\n"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Shape is: 22 x 768\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Iyk_kui98ZGG",
        "outputId": "b7b3816f-83fc-4f15-bdaa-5931b530cc9d"
      },
      "source": [
        "# `hidden_states` has shape [13 x 1 x 22 x 768]\n",
        "\n",
        "# `token_vecs` is a tensor with shape [22 x 768]\n",
        "token_vecs = hidden_states[-2][0]\n",
        "\n",
        "# Calculate the average of all 22 token vectors.\n",
        "sentence_embedding = torch.mean(token_vecs, dim=0)\n",
        "print (\"Our final sentence embedding vector of shape:\", sentence_embedding.size())"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Our final sentence embedding vector of shape: torch.Size([768])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xhq87G_YMLbW"
      },
      "source": [
        "#Confirming contextually dependent vectors\n",
        "\n",
        "To confirm that the value of these vectors are in fact contextually dependent, let’s look at the different instances of the word “bank” in our example sentence:\n",
        "\n",
        "\"“After stealing money from the bank vault, the bank robber was seen fishing on the Mississippi river bank.”\"\n",
        "\n",
        "Bank is located.\n",
        "\n",
        "They are at 6, 10, and 19.\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bgus_-LS8a9g",
        "outputId": "4583e23d-0181-41fa-c6cb-68e1bdb719d2"
      },
      "source": [
        "for i, token_str in enumerate(tokenized_text):\n",
        "  print (i, token_str)"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0 [CLS]\n",
            "1 after\n",
            "2 stealing\n",
            "3 money\n",
            "4 from\n",
            "5 the\n",
            "6 bank\n",
            "7 vault\n",
            "8 ,\n",
            "9 the\n",
            "10 bank\n",
            "11 robber\n",
            "12 was\n",
            "13 seen\n",
            "14 fishing\n",
            "15 on\n",
            "16 the\n",
            "17 mississippi\n",
            "18 river\n",
            "19 bank\n",
            "20 .\n",
            "21 [SEP]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vXw4WNnQ8dpO",
        "outputId": "e9893915-d4b8-4062-dd31-9e732f89212d"
      },
      "source": [
        "print('First 5 vector values for each instance of \"bank\".')\n",
        "print('')\n",
        "print(\"bank vault   \", str(token_vecs_sum[6][:5]))\n",
        "print(\"bank robber  \", str(token_vecs_sum[10][:5]))\n",
        "print(\"river bank   \", str(token_vecs_sum[19][:5]))"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "First 5 vector values for each instance of \"bank\".\n",
            "\n",
            "bank vault    tensor([ 3.3596, -2.9805, -1.5421,  0.7065,  2.0031])\n",
            "bank robber   tensor([ 2.7359, -2.5577, -1.3094,  0.6797,  1.6633])\n",
            "river bank    tensor([ 1.5266, -0.8895, -0.5152, -0.9298,  2.8334])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l8Pgc7dZ8gKF"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}