BERT-model.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "orig_nbformat": 2,
    "colab": {
      "name": "Question_3.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "b80600d0c6e34498b6058c2043dcf1b5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_dd087c0a91ac44cbacae1ade1f7129dd",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4d068eeca50a464f99e2df03a1068711",
              "IPY_MODEL_93a69cbdd7ec49c6b124a8860097621a"
            ]
          }
        },
        "dd087c0a91ac44cbacae1ade1f7129dd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4d068eeca50a464f99e2df03a1068711": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_11beb319c00e46aa86b300e22fd16f6e",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ed179a4fc3cb473db43d27fb82f48240"
          }
        },
        "93a69cbdd7ec49c6b124a8860097621a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_87e65a0dc6de4b80a66f219dba8be394",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:00&lt;00:00, 308kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f72d3df4ed2841ad9fb5c70d20362c20"
          }
        },
        "11beb319c00e46aa86b300e22fd16f6e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ed179a4fc3cb473db43d27fb82f48240": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "87e65a0dc6de4b80a66f219dba8be394": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f72d3df4ed2841ad9fb5c70d20362c20": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0fa5c0384f904912907c6cea29fa1ac4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_663fc6642e3740acb0556b214a8ab9ce",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2c66f7c89f97464dbe8aa5b2352eeb0a",
              "IPY_MODEL_c225cd2020da460aadd62a5675910f09"
            ]
          }
        },
        "663fc6642e3740acb0556b214a8ab9ce": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2c66f7c89f97464dbe8aa5b2352eeb0a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_451407260dd94051af3663d90004de7f",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 433,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 433,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a85a67e4ef2f4dc699c1ef6c39e99581"
          }
        },
        "c225cd2020da460aadd62a5675910f09": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4ab5ec4d07064505980dbbe5c7f123d4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 433/433 [00:16&lt;00:00, 25.6B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_01094799a52a46f29f9504aa26ecbc84"
          }
        },
        "451407260dd94051af3663d90004de7f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a85a67e4ef2f4dc699c1ef6c39e99581": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4ab5ec4d07064505980dbbe5c7f123d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "01094799a52a46f29f9504aa26ecbc84": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "884c6223698a4205b0c5d896958def45": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ae26a72556df4a9594774492019b0a16",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5cd9c4e16b36485480e65fb533b7c495",
              "IPY_MODEL_e228b477aeaf4ed69c2be350554c1eb2"
            ]
          }
        },
        "ae26a72556df4a9594774492019b0a16": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5cd9c4e16b36485480e65fb533b7c495": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8b2bf6952a5645648eff3bc30739bc6c",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 440473133,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 440473133,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_782e4892fdad47ec953208411fac3564"
          }
        },
        "e228b477aeaf4ed69c2be350554c1eb2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1ccddc1238bb424d840ff705dccedf0f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 440M/440M [00:15&lt;00:00, 27.5MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8abdfa0f416340c9b4d5b5e1968e7824"
          }
        },
        "8b2bf6952a5645648eff3bc30739bc6c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "782e4892fdad47ec953208411fac3564": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1ccddc1238bb424d840ff705dccedf0f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8abdfa0f416340c9b4d5b5e1968e7824": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Myrto-Iglezou/AI2-project4/blob/master/Question_3.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eOoUR08ESUJj"
      },
      "source": [
        "## YΣ19 Artificial Intelligence II\n",
        "# Homework 4\n",
        "\n",
        "### Iglezou Myrto - 111520170038"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4Ef8dyqplmv_"
      },
      "source": [
        "# Project Description\r\n",
        "\r\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EjtCURBonpls"
      },
      "source": [
        "Build a BERT-based model which returns “an answer”, given a user question and a\r\n",
        "passage which includes the answer of the question. For this question answering task, we\r\n",
        "will use the SQuAD 2.0 dataset. We will start with the BERT-base pretrained model “bert-base-uncased”\r\n",
        "and fine-tune it to have a question answering task.\r\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lX2R8NAZcZpK"
      },
      "source": [
        "# **Question 3** "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qnA7Kn6DTGw3"
      },
      "source": [
        "### Load SCuAD 2.0 dataset from drive"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G-ruuo--TQVI",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "844ea17c-b55a-495c-8590-a64c121a1358"
      },
      "source": [
        "import io\r\n",
        "import os\r\n",
        "from google.colab import drive\r\n",
        "import pandas as pd \r\n",
        "import numpy as np\r\n",
        "import json\r\n",
        "import sys\r\n",
        "\r\n",
        "drive.mount('/content/drive',force_remount=True)\r\n",
        "\r\n",
        "sys.path.append('/content/drive/My Drive/')\r\n",
        "\r\n",
        "!cp -r \"/content/drive/My Drive/train-v2.0.json\" '/content/'\r\n",
        "!cp -r \"/content/drive/My Drive/dev-v2.0.json\" '/content/'\r\n",
        "!cp -r \"/content/drive/My Drive/utils_squad.py\" '/content/'\r\n",
        "!cp -r \"/content/drive/My Drive/utils_squad_evaluate.py\" '/content/'\r\n",
        "\r\n",
        "train_file = '/content/train-v2.0.json'\r\n",
        "validation_file = '/content/dev-v2.0.json'\r\n",
        "\r\n",
        "with open(train_file) as f:\r\n",
        "    raw_train_data = json.load(f)\r\n",
        "with open(validation_file) as f:\r\n",
        "    raw_val_data = json.load(f)"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D6A6hZwbKdTw"
      },
      "source": [
        "Install transformers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dFi3U07nzGJ-"
      },
      "source": [
        "%%capture\r\n",
        "!pip install transformers"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hBgnJihHN50K"
      },
      "source": [
        "Create the tokenizer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uTmTzd0WzLCu",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67,
          "referenced_widgets": [
            "b80600d0c6e34498b6058c2043dcf1b5",
            "dd087c0a91ac44cbacae1ade1f7129dd",
            "4d068eeca50a464f99e2df03a1068711",
            "93a69cbdd7ec49c6b124a8860097621a",
            "11beb319c00e46aa86b300e22fd16f6e",
            "ed179a4fc3cb473db43d27fb82f48240",
            "87e65a0dc6de4b80a66f219dba8be394",
            "f72d3df4ed2841ad9fb5c70d20362c20"
          ]
        },
        "outputId": "856b0251-1d06-4c76-8887-7285e83d2ac4"
      },
      "source": [
        "import torch\r\n",
        "from transformers import BertTokenizer, BertForSequenceClassification\r\n",
        "from transformers import BertForQuestionAnswering\r\n",
        "from tokenizers import BertWordPieceTokenizer\r\n",
        "\r\n",
        "# Load pre-trained model tokenizer (vocabulary)\r\n",
        "slow_tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\")\r\n",
        "save_path = \"bert_base_uncased/\"\r\n",
        "if not os.path.exists(save_path):\r\n",
        "    os.makedirs(save_path)\r\n",
        "slow_tokenizer.save_pretrained(save_path)\r\n",
        "\r\n",
        "# Load the fast tokenizer from saved file\r\n",
        "tokenizer = BertWordPieceTokenizer(\"bert_base_uncased/vocab.txt\", lowercase=True)\r\n",
        "max_len = 384\r\n"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b80600d0c6e34498b6058c2043dcf1b5",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CACrKo9MN9-T"
      },
      "source": [
        "Create a class to save the data form sQuAD"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qyZR4BnGQKYY"
      },
      "source": [
        "class SquadSample:\r\n",
        "  \r\n",
        "    def __init__(self, context, question, basic_answer, more_answers, start_idx):\r\n",
        "        self.context = context\r\n",
        "        self.question = question\r\n",
        "        self.basic_answer = basic_answer\r\n",
        "        self.more_answers = more_answers\r\n",
        "        self.start_idx = start_idx\r\n",
        "        self.end_idx = None\r\n",
        "        self.start_idx_token = start_idx\r\n",
        "        self.end_idx_token = None\r\n",
        "        self.offsets = None\r\n",
        "        self.input_ids = None \r\n",
        "        self.attention_mask = None\r\n",
        "        self.token_type_ids = None\r\n",
        "        self.validExample = True\r\n",
        "\r\n",
        "    def preprocess(self): # function to preprocess data\r\n",
        "       # Clean context, answer and question\r\n",
        "        self.context = \" \".join(str(self.context).split())\r\n",
        "        self.question = \" \".join(str(self.question).split())\r\n",
        "        \r\n",
        "        contextTokenizer =  tokenizer.encode(self.context)\r\n",
        "\r\n",
        "        if self.basic_answer is not None: # in case we have an answer\r\n",
        "\r\n",
        "          self.basic_answer = \" \".join(str(self.basic_answer).split())\r\n",
        "          #Calculate end_idx\r\n",
        "          self.end_idx = self.start_idx + len(self.basic_answer)\r\n",
        "          if (self.end_idx >=len(self.context)):\r\n",
        "              self.validExample= False\r\n",
        "              return\r\n",
        "\r\n",
        "          #find characters of context that are part of answer\r\n",
        "          is_part_of_answer = [0]*len(self.context)\r\n",
        "          for i in range (self.start_idx, self.end_idx):\r\n",
        "              is_part_of_answer[i] = 1\r\n",
        "\r\n",
        "          \r\n",
        "          #find index of token that corresponds to start and the end of the answer\r\n",
        "          answer_id_token=[]\r\n",
        "          for idx, (start,end) in enumerate(contextTokenizer.offsets):\r\n",
        "              if (sum(is_part_of_answer[start:end]) >0 ):\r\n",
        "                  answer_id_token.append(idx)\r\n",
        "          #data to predict\r\n",
        "          if len(answer_id_token) == 0 :\r\n",
        "              self.validExample=False\r\n",
        "              return         \r\n",
        "          self.start_idx_token = answer_id_token[0]\r\n",
        "          self.end_idx_token = answer_id_token[-1]\r\n",
        "        self.offsets = contextTokenizer.offsets\r\n",
        "        \r\n",
        "        # work on question\r\n",
        "        questionTokinizer  = tokenizer.encode(self.question)\r\n",
        "\r\n",
        "        #Create model's inputs \r\n",
        "        self.input_ids = contextTokenizer.ids + questionTokinizer.ids[1:]\r\n",
        "        self.attention_mask = [1] * len (self.input_ids)\r\n",
        "        self.token_type_ids = [0] * len(contextTokenizer.ids) + [1]*len(questionTokinizer.ids[1:])  \r\n",
        "\r\n",
        "        # fix padding\r\n",
        "        padding_length = max_len - len(self.input_ids)\r\n",
        "        if padding_length > 0:\r\n",
        "            self.input_ids = self.input_ids + ([0] * padding_length)\r\n",
        "            self.attention_mask = self.attention_mask + ([0] * padding_length)\r\n",
        "            self.token_type_ids = self.token_type_ids + ([0] * padding_length)\r\n",
        "        elif padding_length < 0:\r\n",
        "            self.validExample= False \r\n",
        "            return "
      ],
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Aq0fUTFkOg39"
      },
      "source": [
        "Function that helps save data from json files"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-kLAD7_muzhV"
      },
      "source": [
        "def create_squad_examples(raw_data):\r\n",
        "    squad_examples = []\r\n",
        "    for item in raw_data[\"data\"]:\r\n",
        "        for para in item[\"paragraphs\"]:\r\n",
        "            context = para[\"context\"]\r\n",
        "            for qa in para[\"qas\"]:\r\n",
        "                question = qa[\"question\"]\r\n",
        "                if qa[\"answers\"]:\r\n",
        "                  answer_text = qa[\"answers\"][0][\"text\"]\r\n",
        "                  all_answers = [_[\"text\"] for _ in qa[\"answers\"]]\r\n",
        "                  start_char_idx = qa[\"answers\"][0][\"answer_start\"]\r\n",
        "                  #context, question, basic_answer, more_answers, start_idx\r\n",
        "                  squad_eg = SquadSample(context,question, answer_text, all_answers, start_char_idx)\r\n",
        "                else:\r\n",
        "                  squad_eg = SquadSample(context,question, None, None, None) #context, question\r\n",
        "                squad_eg.preprocess()\r\n",
        "                squad_examples.append(squad_eg)\r\n",
        "    return squad_examples"
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7nsI6692O6Jl"
      },
      "source": [
        "Function that create ttwo dictionaries, one for the input and one for the target "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MdfWynK3uz9F"
      },
      "source": [
        "def create_inputs_targets(squad_examples):\r\n",
        "    dataset_dict = {\r\n",
        "        \"input_ids\" : [],\r\n",
        "        \"attention_mask\" : [],\r\n",
        "        \"token_type_ids\" : [],\r\n",
        "        \"start_idx_token\" : [],\r\n",
        "        \"end_idx_token\" : []\r\n",
        "    }\r\n",
        "    for item in squad_examples:\r\n",
        "        if item.validExample is True:\r\n",
        "            for key in dataset_dict:\r\n",
        "                dataset_dict[key].append(getattr(item, key))\r\n",
        "    for key in dataset_dict:\r\n",
        "        dataset_dict[key] = np.array(dataset_dict[key],dtype=np.float16)\r\n",
        "    x = [dataset_dict[\"input_ids\"], dataset_dict[\"attention_mask\"], dataset_dict[\"token_type_ids\"]]\r\n",
        "    y = [dataset_dict[\"start_idx_token\"], dataset_dict[\"end_idx_token\"]]\r\n",
        "    return x, y   "
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ev1vzsRcU6Jx"
      },
      "source": [
        "data = create_squad_examples(raw_train_data)  # save the data for the training set\r\n",
        "\r\n",
        "val_data = create_squad_examples(raw_val_data)  # save the data for the validation set"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BJud54kGPbZE"
      },
      "source": [
        "Check what is saved"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ug8SwSkZ_uTB",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 202
        },
        "outputId": "721b5153-32eb-4f24-9e8f-d414ea93e6bb"
      },
      "source": [
        "train_data = pd.DataFrame.from_records([vars(line) for line in data])\r\n",
        "train_data[[\"context\",\"question\",\"basic_answer\"]].head()"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>context</th>\n",
              "      <th>question</th>\n",
              "      <th>basic_answer</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Beyoncé Giselle Knowles-Carter (/biːˈjɒnseɪ/ b...</td>\n",
              "      <td>When did Beyonce start becoming popular?</td>\n",
              "      <td>in the late 1990s</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Beyoncé Giselle Knowles-Carter (/biːˈjɒnseɪ/ b...</td>\n",
              "      <td>What areas did Beyonce compete in when she was...</td>\n",
              "      <td>singing and dancing</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Beyoncé Giselle Knowles-Carter (/biːˈjɒnseɪ/ b...</td>\n",
              "      <td>When did Beyonce leave Destiny's Child and bec...</td>\n",
              "      <td>2003</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Beyoncé Giselle Knowles-Carter (/biːˈjɒnseɪ/ b...</td>\n",
              "      <td>In what city and state did Beyonce grow up?</td>\n",
              "      <td>Houston, Texas</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Beyoncé Giselle Knowles-Carter (/biːˈjɒnseɪ/ b...</td>\n",
              "      <td>In which decade did Beyonce become famous?</td>\n",
              "      <td>late 1990s</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                             context  ...         basic_answer\n",
              "0  Beyoncé Giselle Knowles-Carter (/biːˈjɒnseɪ/ b...  ...    in the late 1990s\n",
              "1  Beyoncé Giselle Knowles-Carter (/biːˈjɒnseɪ/ b...  ...  singing and dancing\n",
              "2  Beyoncé Giselle Knowles-Carter (/biːˈjɒnseɪ/ b...  ...                 2003\n",
              "3  Beyoncé Giselle Knowles-Carter (/biːˈjɒnseɪ/ b...  ...       Houston, Texas\n",
              "4  Beyoncé Giselle Knowles-Carter (/biːˈjɒnseɪ/ b...  ...           late 1990s\n",
              "\n",
              "[5 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "grzWupQh4BIk"
      },
      "source": [
        "x_train, y_train = create_inputs_targets(data)  # split the training data to input and target\r\n",
        "\r\n",
        "x_eval, y_eval = create_inputs_targets(val_data)  # split the validation data to input and target"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fThhkIjQk31B"
      },
      "source": [
        "doc_stride = 64\r\n",
        "max_seq_length = 128\r\n",
        "max_query_length = 32\r\n",
        "batch_size = 16"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FFkXwvpOP0dX"
      },
      "source": [
        "### Create Datasets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q8wpGNktunPQ"
      },
      "source": [
        "import torch\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "\n",
        "# Convert to Tensors and build dataset\n",
        "train_data = TensorDataset(torch.tensor(x_train[0], dtype=torch.int64),\n",
        "                           torch.tensor(x_train[1], dtype=torch.float),\n",
        "                           torch.tensor(x_train[2], dtype=torch.int64),\n",
        "                           torch.tensor(y_train[0], dtype=torch.int64),\n",
        "                           torch.tensor(y_train[1], dtype=torch.int64))\n",
        "\n",
        "train_sampler = RandomSampler(train_data)\n",
        "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s0bo33FHvcL3"
      },
      "source": [
        "# Convert to Tensors and build dataset\r\n",
        "train_sampler = RandomSampler(train_data)\r\n",
        "train_data_loader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\r\n",
        "eval_data = TensorDataset(torch.tensor(x_eval[0], dtype=torch.int64),\r\n",
        "                          torch.tensor(x_eval[1], dtype=torch.float),\r\n",
        "                          torch.tensor(x_eval[2], dtype=torch.int64),\r\n",
        "                          torch.tensor(y_eval[0], dtype=torch.int64),\r\n",
        "                          torch.tensor(y_eval[1], dtype=torch.int64))\r\n",
        "\r\n",
        "eval_sampler = SequentialSampler(eval_data)\r\n",
        "validation_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=batch_size)"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "752yxQ8PP8BL"
      },
      "source": [
        "### Check for GPU availability"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zIt22mNknCAN",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "45bbaaae-fd31-4216-e560-270f6d86195e"
      },
      "source": [
        "import torch\r\n",
        "# First checking if GPU is available\r\n",
        "train_on_gpu=torch.cuda.is_available()\r\n",
        "\r\n",
        "if(train_on_gpu):\r\n",
        "    print('Training on GPU.')\r\n",
        "    device = 'cuda'\r\n",
        "else:\r\n",
        "    print('No GPU available, training on CPU.')\r\n",
        "    device = 'cpu'"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training on GPU.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1hCGNA6qQCjV"
      },
      "source": [
        "### Initialize model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rlsIqAKRsrLE",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 223,
          "referenced_widgets": [
            "0fa5c0384f904912907c6cea29fa1ac4",
            "663fc6642e3740acb0556b214a8ab9ce",
            "2c66f7c89f97464dbe8aa5b2352eeb0a",
            "c225cd2020da460aadd62a5675910f09",
            "451407260dd94051af3663d90004de7f",
            "a85a67e4ef2f4dc699c1ef6c39e99581",
            "4ab5ec4d07064505980dbbe5c7f123d4",
            "01094799a52a46f29f9504aa26ecbc84",
            "884c6223698a4205b0c5d896958def45",
            "ae26a72556df4a9594774492019b0a16",
            "5cd9c4e16b36485480e65fb533b7c495",
            "e228b477aeaf4ed69c2be350554c1eb2",
            "8b2bf6952a5645648eff3bc30739bc6c",
            "782e4892fdad47ec953208411fac3564",
            "1ccddc1238bb424d840ff705dccedf0f",
            "8abdfa0f416340c9b4d5b5e1968e7824"
          ]
        },
        "outputId": "7d5fd80e-eb53-4a40-9d45-67c390756f01"
      },
      "source": [
        "model = BertForQuestionAnswering.from_pretrained('bert-base-uncased').to(device=device)\r\n",
        "param_optimizer = list(model.named_parameters())"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0fa5c0384f904912907c6cea29fa1ac4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=433.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "884c6223698a4205b0c5d896958def45",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=440473133.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForQuestionAnswering: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
            "- This IS expected if you are initializing BertForQuestionAnswering from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForQuestionAnswering from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForQuestionAnswering were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['qa_outputs.weight', 'qa_outputs.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kbfGuV8NtVkF"
      },
      "source": [
        "no_decay = ['bias', 'gamma', 'beta']\r\n",
        "optimizer_grouped_parameters = [\r\n",
        "    {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],\r\n",
        "     'weight_decay_rate': 0.01},\r\n",
        "    {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],\r\n",
        "     'weight_decay_rate': 0.0}\r\n",
        "]\r\n",
        "\r\n",
        "optimizer = torch.optim.Adam(lr=1e-5, betas=(0.9, 0.98), eps=1e-9, params=optimizer_grouped_parameters)"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mQCQ__PIQPaL"
      },
      "source": [
        "### Training model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fLT084mGQJEM"
      },
      "source": [
        "Function for text normalization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g7n6yBDTCTZV"
      },
      "source": [
        "import string\r\n",
        "import re\r\n",
        "\r\n",
        "def normalize_text(text):\r\n",
        "    text = text.lower()\r\n",
        "    text = \"\".join(ch for ch in text if ch not in set(string.punctuation))\r\n",
        "    regex = re.compile(r\"\\b(a|an|the)\\b\", re.UNICODE)\r\n",
        "    text = re.sub(regex, \" \", text)\r\n",
        "    text = \" \".join(text.split())\r\n",
        "    return text"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qjScNs3OuF0L",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b6ec73af-aa5b-44cd-d909-c824314b15d2"
      },
      "source": [
        "epochs = 1\r\n",
        "\r\n",
        "for epoch in range(1, epochs + 1):\r\n",
        "    # ============================================ TRAINING ============================================================\r\n",
        "    print(\"Training epoch \", str(epoch))\r\n",
        "   \r\n",
        "    model.train()\r\n",
        "    tr_loss = 0\r\n",
        "    nb_tr_steps = 0\r\n",
        "    for step, batch in enumerate(train_dataloader):\r\n",
        "        batch = tuple(t.to(device) for t in batch)\r\n",
        "\r\n",
        "        inputs = {'input_ids':       batch[0],\r\n",
        "                  'attention_mask':  batch[1], \r\n",
        "                  'token_type_ids':  batch[2],  \r\n",
        "                  'start_positions': batch[3], \r\n",
        "                  'end_positions':   batch[4]}\r\n",
        "\r\n",
        "        optimizer.zero_grad()\r\n",
        "\r\n",
        "        outputs = model(**inputs)\r\n",
        "        loss = outputs[0]\r\n",
        "\r\n",
        "        loss.backward()\r\n",
        "        optimizer.step()\r\n",
        "        tr_loss += loss.item()\r\n",
        "        nb_tr_steps += 1\r\n",
        "\r\n",
        "    print(f\"\\nTraining loss={tr_loss / nb_tr_steps:.4f}\")\r\n",
        "\r\n",
        "    # ============================================ VALIDATION ==========================================================\r\n",
        "    model.eval()\r\n",
        "    currect_query = 0\r\n",
        "    correct_ans = 0\r\n",
        "    valid_examples = [x for x in val_data if x.validExample is True]\r\n",
        "    for batch in validation_dataloader:\r\n",
        "        batch = tuple(t.to(device) for t in batch)\r\n",
        "        \r\n",
        "        input_ids, attention_mask, token_type_ids, start_positions, end_positions = batch\r\n",
        "\r\n",
        "        with torch.no_grad():\r\n",
        "          start_logits, end_logits =  model(input_ids=input_ids,\r\n",
        "                                              attention_mask=attention_mask,\r\n",
        "                                              token_type_ids=token_type_ids, return_dict=False)\r\n",
        "          \r\n",
        "          pred_start, pred_end = start_logits.detach().cpu().numpy(), end_logits.detach().cpu().numpy()\r\n",
        "\r\n",
        "        for idx, (start, end) in enumerate(zip(pred_start, pred_end)):\r\n",
        "            squad_eg = valid_examples[currect_query]\r\n",
        "            currect_query += 1\r\n",
        "            offsets = squad_eg.offsets\r\n",
        "            start = np.argmax(start)\r\n",
        "            end = np.argmax(end)\r\n",
        "            if start >= len(offsets):\r\n",
        "                continue\r\n",
        "            pred_char_start = offsets[start][0]\r\n",
        "            if end < len(offsets):\r\n",
        "                pred_char_end = offsets[end][1]\r\n",
        "                pred_ans = squad_eg.context[pred_char_start:pred_char_end]\r\n",
        "            else:\r\n",
        "                pred_ans = squad_eg.context[pred_char_start:]\r\n",
        "            normalized_pred_ans = normalize_text(pred_ans)\r\n",
        "            normalized_true_ans = [normalize_text(x) for x in squad_eg.more_answers]\r\n",
        "            if normalized_pred_ans in normalized_true_ans:\r\n",
        "                correct_ans += 1\r\n",
        "    acc = correct_ans / len(y_eval[0])\r\n",
        "\r\n",
        "    print(f\"\\nAccuracy score={acc:.2f}\\n\")\r\n",
        "    print(\"-----------------------------------------\")"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training epoch  1\n",
            "\n",
            "Training loss=1.9051\n",
            "\n",
            "Accuracy score=0.57\n",
            "\n",
            "-----------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XqrNzdjFqg0I"
      },
      "source": [
        "### Testing model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l4gYupzjXreU",
        "outputId": "39b8ad89-6008-420c-bdf1-b3d1aea654c3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "# ============================================ TESTING ==========================================================\r\n",
        "data = {\"data\":\r\n",
        "    [\r\n",
        "        {\"title\": \"Tesla's Biography\",\r\n",
        "         \"paragraphs\": [\r\n",
        "             {\r\n",
        "                 \"context\": \"Nikola Tesla was a Serbian-American inventor, electrical engineer, mechanical engineer, \"\r\n",
        "                            \"and futurist best known for his contributions to the design of the modern alternating \"\r\n",
        "                            \"current (AC) electricity supply system. Born and raised in the Austrian Empire,Tesla \"\r\n",
        "                            \"studied engineering and physics in the 1870s without receiving a degree. In 1884, he \"\r\n",
        "                            \"moved to United States. In 1887, Tesla developed an induction motor that ran on alternating \"\r\n",
        "                            \"current (AC), a power system format that was rapidly expanding in Europe and the United \"\r\n",
        "                            \"States because of its advantages in long-distance, high-voltage transmission. Tesla became \"\r\n",
        "                            \"a vegetarian in his later years, living on only milk, bread, honey, and vegetable juices. \"\r\n",
        "                            \"On 7 January 1943, at the age of 86, Tesla died alone in Room 3327 of the Hotel New Yorker. \"\r\n",
        "                            \"Tesla wrote a number of books and articles for magazines and journals. Among his books are \"\r\n",
        "                            \"My Inventions: The Autobiography of Nikola Tesla, compiled and edited by Ben Johnston \"\r\n",
        "                            \"in 1983 from a series of 1919 magazine articles by Tesla which were republished in 1977. \"\r\n",
        "                            \"Tesla's legacy has endured in books, films, radio, TV, music, live theater, comics, and \"\r\n",
        "                            \"video games. The impact of the technologies invented or envisioned by Tesla is a recurring \"\r\n",
        "                            \"theme in several types of science fiction. \",\r\n",
        "                 \"qas\": [\r\n",
        "                     {\"question\": \"When did Tesla becοme a vegetarian?\",\r\n",
        "                      \"id\": \"Q1\",\r\n",
        "                      \"answers\":\"\"\r\n",
        "                      },\r\n",
        "                     {\"question\": \"When did Tesla move to United States ?\",\r\n",
        "                      \"id\": \"Q2\",\r\n",
        "                      \"answers\":\"\"\r\n",
        "                      },\r\n",
        "                     {\"question\": \"What year did Tesla die?\",\r\n",
        "                      \"id\": \"Q3\",\r\n",
        "                      \"answers\":\"\"\r\n",
        "                      },\r\n",
        "                     {\"question\": \"Who edited the book My Inventions: The Autobiography of Nikola Tesla?\",\r\n",
        "                      \"id\": \"Q4\",\r\n",
        "                      \"answers\":\"\"\r\n",
        "                      },\r\n",
        "                     {\"question\": \"In what age did Tesla died?\",\r\n",
        "                      \"id\": \"Q5\",\r\n",
        "                      \"answers\":\"\"\r\n",
        "                      },\r\n",
        "                     {\"question\": \"Who developed an induction motor?\",\r\n",
        "                      \"id\": \"Q6\",\r\n",
        "                      \"answers\":\"\"\r\n",
        "                      },\r\n",
        "                     {\"question\": \"Where Tesla was born?\",\r\n",
        "                      \"id\": \"Q7\",\r\n",
        "                      \"answers\":\"\"\r\n",
        "                      },\r\n",
        "                     {\"question\": \"What did Tesla study?\",\r\n",
        "                      \"id\": \"Q8\",\r\n",
        "                      \"answers\":\"\"\r\n",
        "                      },\r\n",
        "                 ]}]}]}\r\n",
        "\r\n",
        "model.eval()\r\n",
        "test_samples = create_squad_examples(data)\r\n",
        "x_test, _ = create_inputs_targets(test_samples)\r\n",
        "pred_start, pred_end = model(torch.tensor(x_test[0], dtype=torch.int64, device=device),\r\n",
        "                             torch.tensor(x_test[1], dtype=torch.float, device=device),\r\n",
        "                             torch.tensor(x_test[2], dtype=torch.int64, device=device), return_dict=False)\r\n",
        "pred_start, pred_end = pred_start.detach().cpu().numpy(), pred_end.detach().cpu().numpy()\r\n",
        "for idx, (start, end) in enumerate(zip(pred_start, pred_end)):\r\n",
        "    test_sample = test_samples[idx]\r\n",
        "    offsets = test_sample.offsets\r\n",
        "    start = np.argmax(start)\r\n",
        "    end = np.argmax(end)\r\n",
        "    pred_ans = None\r\n",
        "    if start >= len(offsets):\r\n",
        "        continue\r\n",
        "    pred_char_start = offsets[start][0]\r\n",
        "    if end < len(offsets):\r\n",
        "        pred_ans = test_sample.context[pred_char_start:offsets[end][1]]\r\n",
        "    else:\r\n",
        "        pred_ans = test_sample.context[pred_char_start:]\r\n",
        "    print(\"Q: \" + test_sample.question)\r\n",
        "    print(\"A: \" + pred_ans)\r\n",
        "    print(\"----------------------------------------\\n\")"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Q: When did Tesla becοme a vegetarian?\n",
            "A: his later years\n",
            "----------------------------------------\n",
            "\n",
            "Q: When did Tesla move to United States ?\n",
            "A: 1884\n",
            "----------------------------------------\n",
            "\n",
            "Q: What year did Tesla die?\n",
            "A: 1943\n",
            "----------------------------------------\n",
            "\n",
            "Q: Who edited the book My Inventions: The Autobiography of Nikola Tesla?\n",
            "A: Ben Johnston\n",
            "----------------------------------------\n",
            "\n",
            "Q: In what age did Tesla died?\n",
            "A: 86\n",
            "----------------------------------------\n",
            "\n",
            "Q: Who developed an induction motor?\n",
            "A: Tesla\n",
            "----------------------------------------\n",
            "\n",
            "Q: Where Tesla was born?\n",
            "A: Austrian Empire\n",
            "----------------------------------------\n",
            "\n",
            "Q: What did Tesla study?\n",
            "A: engineering and physics\n",
            "----------------------------------------\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fu9jcIFM-eMi"
      },
      "source": [
        "## References\r\n",
        "\r\n",
        "\r\n",
        "\r\n",
        "*   https://github.com/nlpyang/pytorch-transformers/tree/master/examples\r\n",
        "*  https://github.com/flogothetis/SQuAD-QueryAnswering-BERT-Keras/blob/main/SQuAD_QuestionAnswering_Bert.ipynb\r\n",
        "*  https://github.com/dredwardhyde/bert-examples/blob/main/bert_squad_pytorch.py?fbclid=IwAR1VGhZx6MsVlOha3lDX_uC8PASSDu9ECKceD2XCHGSetKhldgay0F8SirY\r\n",
        "* https://colab.research.google.com/drive/1Zp2_Uka8oGDYsSe5ELk-xz6wIX8OIkB7?fbclid=IwAR1zl-nOBSOdYA4H-WY-ba6AJ--hRHM2OZhgK3DrQ1SLfavln5M-k-r4jJ4#scrollTo=j3_CAQUf2asD\r\n",
        "\r\n"
      ]
    }
  ]
}