Transformers_in_pytorch.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyMbSwvNWiqm5C3a9ZmBT0B8",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "261ba7d1591d4c0cb9ba4a0c693689d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_78170014c2f9401fb0423d7386016a43",
              "IPY_MODEL_3b2a6311492f44808c1b98fd2a6267f0",
              "IPY_MODEL_09e9e116fedf482788c38ee3b31f8181"
            ],
            "layout": "IPY_MODEL_8132cf3198e04ef0923a10b69b50b482"
          }
        },
        "78170014c2f9401fb0423d7386016a43": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f2d760da92b94e29b0fda376f4fc2cc0",
            "placeholder": "​",
            "style": "IPY_MODEL_5ecad9dbd3a04e09ba6905e907a30f69",
            "value": "Downloading (…)okenizer_config.json: 100%"
          }
        },
        "3b2a6311492f44808c1b98fd2a6267f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_192ff591db244ff188c83fab5af77025",
            "max": 29,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_32363a859ea9436abd98d6f98785fc77",
            "value": 29
          }
        },
        "09e9e116fedf482788c38ee3b31f8181": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_53a41a265a1747b6b6e918b7d24a665e",
            "placeholder": "​",
            "style": "IPY_MODEL_eedbd1c4a8f9400986868e92152ed430",
            "value": " 29.0/29.0 [00:00&lt;00:00, 888B/s]"
          }
        },
        "8132cf3198e04ef0923a10b69b50b482": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f2d760da92b94e29b0fda376f4fc2cc0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5ecad9dbd3a04e09ba6905e907a30f69": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "192ff591db244ff188c83fab5af77025": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "32363a859ea9436abd98d6f98785fc77": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "53a41a265a1747b6b6e918b7d24a665e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "eedbd1c4a8f9400986868e92152ed430": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "20d969774f8c4a5fa849a016c8d4e900": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7a9b8f0b909a4f9699004c68714db553",
              "IPY_MODEL_683cb16663ca4a58bb63ffd9131c8710",
              "IPY_MODEL_1108b929352c4edfa777a624b6dda618"
            ],
            "layout": "IPY_MODEL_86f5827603fa4651baec9bd4e064a6cb"
          }
        },
        "7a9b8f0b909a4f9699004c68714db553": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_81f343be7e954b0287d5d7870a24973e",
            "placeholder": "​",
            "style": "IPY_MODEL_031974c2d2a24dffb0a43f89c7761349",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "683cb16663ca4a58bb63ffd9131c8710": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f2b1d14879f5493baec35cf4e05c8ac2",
            "max": 411,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2a185b5b038341b6ab34dc30c96b7cd5",
            "value": 411
          }
        },
        "1108b929352c4edfa777a624b6dda618": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2db910cbb5d6400bbcdabd8d404e98a0",
            "placeholder": "​",
            "style": "IPY_MODEL_72b1d461ba2640f1b79c8f41433e66a6",
            "value": " 411/411 [00:00&lt;00:00, 19.9kB/s]"
          }
        },
        "86f5827603fa4651baec9bd4e064a6cb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "81f343be7e954b0287d5d7870a24973e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "031974c2d2a24dffb0a43f89c7761349": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f2b1d14879f5493baec35cf4e05c8ac2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2a185b5b038341b6ab34dc30c96b7cd5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2db910cbb5d6400bbcdabd8d404e98a0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "72b1d461ba2640f1b79c8f41433e66a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3a6a7ac5fb174a19b340b5dbe7fead8c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_048f56ca151244d88c5f6d5c7b780a6f",
              "IPY_MODEL_443282a1b1c14ddf9205740cebe80cc0",
              "IPY_MODEL_2aa47f3440804744aeb84395f4dff92a"
            ],
            "layout": "IPY_MODEL_43bd3e8bf71a4d15b23dfebd60e553fc"
          }
        },
        "048f56ca151244d88c5f6d5c7b780a6f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4514cd18db37473d81e4a82765cb632b",
            "placeholder": "​",
            "style": "IPY_MODEL_3c9c1bc50d464bd895a2b4ff74c1973c",
            "value": "Downloading (…)solve/main/vocab.txt: 100%"
          }
        },
        "443282a1b1c14ddf9205740cebe80cc0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_64db571b882a4f298323d361f9352c19",
            "max": 213450,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_bb169d2ef43a4cfeaf90bb3c7aa2a375",
            "value": 213450
          }
        },
        "2aa47f3440804744aeb84395f4dff92a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f0a99381214740339d8c90198c58130f",
            "placeholder": "​",
            "style": "IPY_MODEL_c9669d95d5584fb584772360943a101d",
            "value": " 213k/213k [00:00&lt;00:00, 4.92MB/s]"
          }
        },
        "43bd3e8bf71a4d15b23dfebd60e553fc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4514cd18db37473d81e4a82765cb632b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3c9c1bc50d464bd895a2b4ff74c1973c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "64db571b882a4f298323d361f9352c19": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bb169d2ef43a4cfeaf90bb3c7aa2a375": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f0a99381214740339d8c90198c58130f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c9669d95d5584fb584772360943a101d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "253daa13e88c49b399baba35264ca402": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0d1a852a8f854a44be114224c6c4c062",
              "IPY_MODEL_d7c3404d9b7f419b82f32586a27e4a83",
              "IPY_MODEL_08ee9f12fdcd4e4fb7738e6c4fa75210"
            ],
            "layout": "IPY_MODEL_b34f99299cfb4a56b1384d2bbeee0ec1"
          }
        },
        "0d1a852a8f854a44be114224c6c4c062": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d202856cb9604263b20f8cd6d08895c8",
            "placeholder": "​",
            "style": "IPY_MODEL_449ef33ac6e441c6aab3c853722aec05",
            "value": "Downloading (…)/main/tokenizer.json: 100%"
          }
        },
        "d7c3404d9b7f419b82f32586a27e4a83": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ae808eb0d3a84f55b6b31e1d20e2a507",
            "max": 435797,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f41489f07ce3417d8d911f292470230a",
            "value": 435797
          }
        },
        "08ee9f12fdcd4e4fb7738e6c4fa75210": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e4e3e954bb1649448dd4dc4dc63a6772",
            "placeholder": "​",
            "style": "IPY_MODEL_e3eb25432dfe4a009e393282338916fa",
            "value": " 436k/436k [00:00&lt;00:00, 693kB/s]"
          }
        },
        "b34f99299cfb4a56b1384d2bbeee0ec1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d202856cb9604263b20f8cd6d08895c8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "449ef33ac6e441c6aab3c853722aec05": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ae808eb0d3a84f55b6b31e1d20e2a507": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f41489f07ce3417d8d911f292470230a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e4e3e954bb1649448dd4dc4dc63a6772": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e3eb25432dfe4a009e393282338916fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "aead216a8d1d475a81e1e5da25630529": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_83a36ead7d704204a26234b0e0f7ad65",
              "IPY_MODEL_013f63f22c684c8c8bb8895ce3d1b2e0",
              "IPY_MODEL_7c1adde1d85a4479a6c3941063310488"
            ],
            "layout": "IPY_MODEL_1423aae4c2c448cd83c8e2091a092218"
          }
        },
        "83a36ead7d704204a26234b0e0f7ad65": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1883467b5c1248e8afc125e8721744d1",
            "placeholder": "​",
            "style": "IPY_MODEL_3d258350145c43d6bdfc25f157e8c709",
            "value": "Downloading builder script: 100%"
          }
        },
        "013f63f22c684c8c8bb8895ce3d1b2e0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c99e9f56c9a24b68a764cf85e9edc302",
            "max": 28751,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c503e1a20bb24c058777ea2f5ed33652",
            "value": 28751
          }
        },
        "7c1adde1d85a4479a6c3941063310488": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_db81134a1aec44acb03500ca4543f8e1",
            "placeholder": "​",
            "style": "IPY_MODEL_2217294ce415481e8363b9d02cf48e32",
            "value": " 28.8k/28.8k [00:00&lt;00:00, 895kB/s]"
          }
        },
        "1423aae4c2c448cd83c8e2091a092218": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1883467b5c1248e8afc125e8721744d1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3d258350145c43d6bdfc25f157e8c709": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c99e9f56c9a24b68a764cf85e9edc302": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c503e1a20bb24c058777ea2f5ed33652": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "db81134a1aec44acb03500ca4543f8e1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2217294ce415481e8363b9d02cf48e32": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9f14a0a8a3944d8f8e941f5202f84c68": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2d6f09cd27d24131b7b5862a9a8e3915",
              "IPY_MODEL_732bff4ae2814025aa8e5ad808f6c982",
              "IPY_MODEL_606be339b67741ebae6c844fae69de33"
            ],
            "layout": "IPY_MODEL_e0ebff0f593f4dfd98a5a22a58129ed2"
          }
        },
        "2d6f09cd27d24131b7b5862a9a8e3915": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_975a171470bd4fb2b93d911f64b68321",
            "placeholder": "​",
            "style": "IPY_MODEL_fa5bf5ad234347768160daaab01f64d5",
            "value": "Downloading metadata: 100%"
          }
        },
        "732bff4ae2814025aa8e5ad808f6c982": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8b9ab61c150044e1b42df799bc03e5dd",
            "max": 28682,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_10cc83a92ce347ac9d018f8d57d81fc1",
            "value": 28682
          }
        },
        "606be339b67741ebae6c844fae69de33": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ba0996fd89ea4b55910f1c466be2ec87",
            "placeholder": "​",
            "style": "IPY_MODEL_041fe4e0e35244ee824c550bdbd2be82",
            "value": " 28.7k/28.7k [00:00&lt;00:00, 1.34MB/s]"
          }
        },
        "e0ebff0f593f4dfd98a5a22a58129ed2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "975a171470bd4fb2b93d911f64b68321": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fa5bf5ad234347768160daaab01f64d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8b9ab61c150044e1b42df799bc03e5dd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "10cc83a92ce347ac9d018f8d57d81fc1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ba0996fd89ea4b55910f1c466be2ec87": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "041fe4e0e35244ee824c550bdbd2be82": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c0e9c152b79d443c92fbd419e5287032": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_640808c3ac784381a45a949f082380e3",
              "IPY_MODEL_b25f03bbb46249f6b1e00eacbd263bca",
              "IPY_MODEL_114972f375bd40e8bb70a1a0545999e5"
            ],
            "layout": "IPY_MODEL_29c66432e32345638ff08bb1bca40889"
          }
        },
        "640808c3ac784381a45a949f082380e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_75cd7ec346a84ccc97d0b5fe8dcfc2be",
            "placeholder": "​",
            "style": "IPY_MODEL_5c6becc1712749bb97b1de48aa87d42f",
            "value": "Downloading readme: 100%"
          }
        },
        "b25f03bbb46249f6b1e00eacbd263bca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cad092cef9db46a88ba5433b625fb4d3",
            "max": 27882,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f68f9db39ac8475290e5da7237ba93df",
            "value": 27882
          }
        },
        "114972f375bd40e8bb70a1a0545999e5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0893c31ac23e4ab38f1357100c20cba3",
            "placeholder": "​",
            "style": "IPY_MODEL_9f772aafa22e4ac7b8975ec4c249ec84",
            "value": " 27.9k/27.9k [00:00&lt;00:00, 1.95MB/s]"
          }
        },
        "29c66432e32345638ff08bb1bca40889": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "75cd7ec346a84ccc97d0b5fe8dcfc2be": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5c6becc1712749bb97b1de48aa87d42f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cad092cef9db46a88ba5433b625fb4d3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f68f9db39ac8475290e5da7237ba93df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0893c31ac23e4ab38f1357100c20cba3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9f772aafa22e4ac7b8975ec4c249ec84": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "36fbc27c01d0450586072e96ec1df67e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9579693b195545e9988118fd3cc6f66e",
              "IPY_MODEL_fc65da9374da43cf9e247d487e2f183a",
              "IPY_MODEL_338a93a4ed1744ca984969f2dc8ecc0c"
            ],
            "layout": "IPY_MODEL_5e6bfcf8cbcf4bf091d6b0f41ed98076"
          }
        },
        "9579693b195545e9988118fd3cc6f66e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d696f191259e462ba846b5c7d114dc19",
            "placeholder": "​",
            "style": "IPY_MODEL_94cee836e00c470eaff060582dd6a805",
            "value": "Downloading data: 100%"
          }
        },
        "fc65da9374da43cf9e247d487e2f183a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_289ad15f9d9e4e61afa95a55709482ac",
            "max": 7439277,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e9e6bfeae8254189b8517e22c6d1bdd4",
            "value": 7439277
          }
        },
        "338a93a4ed1744ca984969f2dc8ecc0c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_da0954563236433db5002d6a8b7b65ff",
            "placeholder": "​",
            "style": "IPY_MODEL_dfe2b1130d134d9da9ada1e5a6daf4a2",
            "value": " 7.44M/7.44M [00:00&lt;00:00, 51.2MB/s]"
          }
        },
        "5e6bfcf8cbcf4bf091d6b0f41ed98076": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d696f191259e462ba846b5c7d114dc19": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "94cee836e00c470eaff060582dd6a805": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "289ad15f9d9e4e61afa95a55709482ac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e9e6bfeae8254189b8517e22c6d1bdd4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "da0954563236433db5002d6a8b7b65ff": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dfe2b1130d134d9da9ada1e5a6daf4a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b36db7a7342d4c2ab63bdb7c7a8eb586": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_33b8ad2509c949a5858899a65277e6d6",
              "IPY_MODEL_d0b47698ce0c481a9e00549160454938",
              "IPY_MODEL_9929ea473808449e8c50149afbffb3a9"
            ],
            "layout": "IPY_MODEL_d7d3fb82f9284cb3a44778be0c7bddf3"
          }
        },
        "33b8ad2509c949a5858899a65277e6d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7879d0a1b30746d88a182579da17afa7",
            "placeholder": "​",
            "style": "IPY_MODEL_966c72e7e9f24b2abf2beef693c58915",
            "value": "Generating train split: 100%"
          }
        },
        "d0b47698ce0c481a9e00549160454938": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1bd0a41737c749ac8062a2c7e30d4283",
            "max": 67349,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_63755d77b99d4f809ecc0420889df29a",
            "value": 67349
          }
        },
        "9929ea473808449e8c50149afbffb3a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5bdcd4f164644d9293cf90c43d5d0568",
            "placeholder": "​",
            "style": "IPY_MODEL_9aadce43516f4100a83d8a908892859a",
            "value": " 67028/67349 [00:08&lt;00:00, 9232.19 examples/s]"
          }
        },
        "d7d3fb82f9284cb3a44778be0c7bddf3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "7879d0a1b30746d88a182579da17afa7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "966c72e7e9f24b2abf2beef693c58915": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1bd0a41737c749ac8062a2c7e30d4283": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "63755d77b99d4f809ecc0420889df29a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5bdcd4f164644d9293cf90c43d5d0568": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9aadce43516f4100a83d8a908892859a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d81a9ddbf4f44c0dbf4dfc9a0aa84f70": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b648b7c85f3041e297125b408f3a09f5",
              "IPY_MODEL_414b053a920b403080116cca0cafdedf",
              "IPY_MODEL_90a9e4398b8443fe835044c1fd9168b9"
            ],
            "layout": "IPY_MODEL_ee793a41dedb4c77a31745f0f7cdd5d2"
          }
        },
        "b648b7c85f3041e297125b408f3a09f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f2f4686760374c7796b21d6ac5430110",
            "placeholder": "​",
            "style": "IPY_MODEL_904f722c8be0491cbdfe5ed859c93fa8",
            "value": "Generating validation split:  87%"
          }
        },
        "414b053a920b403080116cca0cafdedf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_549942a03c1b47c0aac80bad1f8ffee0",
            "max": 872,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_76947130098447c48631d3580375a2e9",
            "value": 872
          }
        },
        "90a9e4398b8443fe835044c1fd9168b9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fc7b52d0fe1b4ed787c0d1c8c3ea87ed",
            "placeholder": "​",
            "style": "IPY_MODEL_fa20464943ce41d2baa08420fa833f10",
            "value": " 757/872 [00:00&lt;00:00, 5647.19 examples/s]"
          }
        },
        "ee793a41dedb4c77a31745f0f7cdd5d2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "f2f4686760374c7796b21d6ac5430110": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "904f722c8be0491cbdfe5ed859c93fa8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "549942a03c1b47c0aac80bad1f8ffee0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "76947130098447c48631d3580375a2e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "fc7b52d0fe1b4ed787c0d1c8c3ea87ed": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fa20464943ce41d2baa08420fa833f10": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0ff204cac2b2450cb4b25111125b4b05": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_cabf04191cb748a6b391d017b05d2a47",
              "IPY_MODEL_406a22231bf24667baec2c5f66bd092e",
              "IPY_MODEL_0ffba5428c13451c80469591cf051e9f"
            ],
            "layout": "IPY_MODEL_d70e886d78a24d4f9574673020e28bdc"
          }
        },
        "cabf04191cb748a6b391d017b05d2a47": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6eea52594be141c6a770ae5d29a92701",
            "placeholder": "​",
            "style": "IPY_MODEL_bc2ef94e2c4f43aeae42650345026664",
            "value": "Generating test split:  41%"
          }
        },
        "406a22231bf24667baec2c5f66bd092e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2c5f085bfbce486aa2f94a04deb675d7",
            "max": 1821,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_05dc68e3c85842549f919f2dfbd00ee6",
            "value": 1821
          }
        },
        "0ffba5428c13451c80469591cf051e9f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_19807e5f36a948e0a8de927a20ad4e97",
            "placeholder": "​",
            "style": "IPY_MODEL_74fac346ddbb482190c66450c146c1ef",
            "value": " 749/1821 [00:00&lt;00:00, 5621.31 examples/s]"
          }
        },
        "d70e886d78a24d4f9574673020e28bdc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "6eea52594be141c6a770ae5d29a92701": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bc2ef94e2c4f43aeae42650345026664": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2c5f085bfbce486aa2f94a04deb675d7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "05dc68e3c85842549f919f2dfbd00ee6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "19807e5f36a948e0a8de927a20ad4e97": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "74fac346ddbb482190c66450c146c1ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3398c91da17c472793574a3c86c63f0e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_21713f2bd5834571a23b2ca87626c34d",
              "IPY_MODEL_2313a9d7236b4dcba4ae206aa85d90fa",
              "IPY_MODEL_0072c675564048ddaa887caf8f509025"
            ],
            "layout": "IPY_MODEL_0e169378ce6a41ba86b04f4534531c68"
          }
        },
        "21713f2bd5834571a23b2ca87626c34d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ad041993bd234b9d894b3b66db224788",
            "placeholder": "​",
            "style": "IPY_MODEL_787a1414d9274b8cbaf97f351a626e98",
            "value": "100%"
          }
        },
        "2313a9d7236b4dcba4ae206aa85d90fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0a9c7ad3f5544510aff1db42a2291b12",
            "max": 3,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f6769f2d24d249f3b3f63ca6fce0c1b8",
            "value": 3
          }
        },
        "0072c675564048ddaa887caf8f509025": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8f6078ac6ba04302bfced59497be94c0",
            "placeholder": "​",
            "style": "IPY_MODEL_b0e692e508844336ba43a22dc3eda1e8",
            "value": " 3/3 [00:00&lt;00:00, 42.35it/s]"
          }
        },
        "0e169378ce6a41ba86b04f4534531c68": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ad041993bd234b9d894b3b66db224788": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "787a1414d9274b8cbaf97f351a626e98": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0a9c7ad3f5544510aff1db42a2291b12": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f6769f2d24d249f3b3f63ca6fce0c1b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8f6078ac6ba04302bfced59497be94c0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b0e692e508844336ba43a22dc3eda1e8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1b42bba5ee964eb29b7985c7a2713a77": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_66e84e5b7ab44557a26ecad20ad7f570",
              "IPY_MODEL_fb2a0b35683b44ee8ed9b255f1342398",
              "IPY_MODEL_0350852ab4bc417e9da41665b6289607"
            ],
            "layout": "IPY_MODEL_5483f91b5780410dbe5f60ecd15a99bd"
          }
        },
        "66e84e5b7ab44557a26ecad20ad7f570": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f34b5f53d83e4e1dadbcde31d507b9cf",
            "placeholder": "​",
            "style": "IPY_MODEL_721242311de741498468f0d5d1dfdaa5",
            "value": "Map:  98%"
          }
        },
        "fb2a0b35683b44ee8ed9b255f1342398": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_47e46aaeb22a462d86e6af0076f7c3af",
            "max": 67349,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a952f3b9073648828113d2f81af77ccd",
            "value": 67349
          }
        },
        "0350852ab4bc417e9da41665b6289607": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_737b9108dae64e0aae2086b835289bfb",
            "placeholder": "​",
            "style": "IPY_MODEL_d376e9b943d5458a87a0005aa88a702d",
            "value": " 66000/67349 [00:06&lt;00:00, 12512.65 examples/s]"
          }
        },
        "5483f91b5780410dbe5f60ecd15a99bd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "f34b5f53d83e4e1dadbcde31d507b9cf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "721242311de741498468f0d5d1dfdaa5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "47e46aaeb22a462d86e6af0076f7c3af": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a952f3b9073648828113d2f81af77ccd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "737b9108dae64e0aae2086b835289bfb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d376e9b943d5458a87a0005aa88a702d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "787574f727d5484889bbc298b250644d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_27bace151ccf4dcdbeb85474f9dc2edc",
              "IPY_MODEL_00c1cadd1a7d4581a9a62995576d0b18",
              "IPY_MODEL_53a83096e7a44570a42734f34c25f223"
            ],
            "layout": "IPY_MODEL_cac44d6cb58e46cb9972afbaf699f5e1"
          }
        },
        "27bace151ccf4dcdbeb85474f9dc2edc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5dbbe6b3557e4fc19079ca8412a35317",
            "placeholder": "​",
            "style": "IPY_MODEL_6d271f007cb445ec8e77c4e53a52e2dd",
            "value": "Map: 100%"
          }
        },
        "00c1cadd1a7d4581a9a62995576d0b18": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2ccd841be81e4c2e97aec55c5dc33e1a",
            "max": 872,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_cd210bd9760c48b08785d227083d710f",
            "value": 872
          }
        },
        "53a83096e7a44570a42734f34c25f223": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a85c5c4270f8442abff4c2a80941742f",
            "placeholder": "​",
            "style": "IPY_MODEL_0257f4602f5f47ab9e20a82cfa04bcb1",
            "value": " 872/872 [00:00&lt;00:00, 6143.31 examples/s]"
          }
        },
        "cac44d6cb58e46cb9972afbaf699f5e1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "5dbbe6b3557e4fc19079ca8412a35317": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6d271f007cb445ec8e77c4e53a52e2dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2ccd841be81e4c2e97aec55c5dc33e1a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cd210bd9760c48b08785d227083d710f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a85c5c4270f8442abff4c2a80941742f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0257f4602f5f47ab9e20a82cfa04bcb1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bd395ab50f5f41c3b379e1d700380a90": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7424f8a3cf6a43558d87981538132a5d",
              "IPY_MODEL_a36ca74068464eae938c62989ae50c18",
              "IPY_MODEL_35feb2c81b08484d98c5233a90878864"
            ],
            "layout": "IPY_MODEL_c963602d1b804aae82ab2091c9315399"
          }
        },
        "7424f8a3cf6a43558d87981538132a5d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e662511e959b41be8f039703a9f240c7",
            "placeholder": "​",
            "style": "IPY_MODEL_5a8f9417648e4d60bbec52f0e07ae696",
            "value": "Map: 100%"
          }
        },
        "a36ca74068464eae938c62989ae50c18": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a7b0d43005fa4428b5c60d699e29823d",
            "max": 1821,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ccb1066309bf4f66a947bdb2009163b0",
            "value": 1821
          }
        },
        "35feb2c81b08484d98c5233a90878864": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2cabf3d925e948eb9d7ef001e3c67ee6",
            "placeholder": "​",
            "style": "IPY_MODEL_98f8ebbbb59340f0adfb89757d3b071c",
            "value": " 1821/1821 [00:00&lt;00:00, 2131.92 examples/s]"
          }
        },
        "c963602d1b804aae82ab2091c9315399": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "e662511e959b41be8f039703a9f240c7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5a8f9417648e4d60bbec52f0e07ae696": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a7b0d43005fa4428b5c60d699e29823d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ccb1066309bf4f66a947bdb2009163b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2cabf3d925e948eb9d7ef001e3c67ee6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "98f8ebbbb59340f0adfb89757d3b071c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Ogunfool/A-Transformer-Based-Model-For-Multivariate-Time-Series-Prediction-Task-Built-from-scratch-in-pytorch/blob/main/Transformers_in_pytorch.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Import Necessary Libraries"
      ],
      "metadata": {
        "id": "sHpmosEqiuGc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import math\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch.utils.data import dataset"
      ],
      "metadata": {
        "id": "fQuItHpD-g2k"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Encoders Transformer Based Network"
      ],
      "metadata": {
        "id": "Vu6Ew29Ni8p_"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Building Transformers from scratch. The modules include:\n",
        "\n",
        "*   Multi-head Attention\n",
        "*   Transformer Block(s)\n",
        "\n",
        "*   Positional Encoding\n",
        "*   Encoder / Decoder\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "je6EpUFIT-6B"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Pytorch Modules - Example.\n",
        "\n",
        "---\n",
        "\n",
        "\n",
        "Base class for all neural network modules.\n",
        "\n",
        "Your models should also subclass this class.\n",
        "\n",
        "Modules can also contain other Modules, allowing to nest them in a tree structure. You can assign the submodules as regular attributes:"
      ],
      "metadata": {
        "id": "jBo6Nj4VjuYH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class Model(nn.Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.conv1 = nn.Conv2d(1, 20, 5)\n",
        "        self.conv2 = nn.Conv2d(20, 20, 5)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = F.relu(self.conv1(x))\n",
        "        return F.relu(self.conv2(x))"
      ],
      "metadata": {
        "id": "KzLFh40jjLzM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Implement Multi-head Attention from Scratch."
      ],
      "metadata": {
        "id": "RoVTsN10Ut_I"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "adMHTPoMT6Lg"
      },
      "outputs": [],
      "source": [
        "# Excerpt form lazy programmer NLP course....\n",
        "# Subclass the torch.nn class\n",
        "class MultiHeadAttention(nn.Module):\n",
        "  def __init__(self, d_k, d_model, no_heads):\n",
        "    super().__init__()\n",
        "    # d_k = key dimension, d_q = query dimension and d_v = value dimension\n",
        "    # d_k and d_q have the same dimension, d_v is also assumed to be d_k\n",
        "    self.d_k = d_k\n",
        "    self.n_heads = no_heads\n",
        "\n",
        "    # input feature size = size of x = (N,T,d_model)\n",
        "    # multi-head vectorized implementation \n",
        "    self.key = nn.Linear(d_model, d_k * no_heads)\n",
        "    self.query = nn.Linear(d_model, d_k * no_heads)\n",
        "    self.value = nn.Linear(d_model, d_k * no_heads)\n",
        "\n",
        "    # final linear layer - To convert N x T x d_v to N x T x d_model\n",
        "    self.fc = nn.Linear(d_k * no_heads, d_model)\n",
        "\n",
        "  def forward(self, x, mask=None):\n",
        "    q = self.query(x) # N x T x (h*d_k)\n",
        "    k = self.key(x)   # N x T x (h*d_k)\n",
        "    v = self.value(x) # N x T x (h*d_v)\n",
        "\n",
        "    N = q.shape[0]\n",
        "    T = q.shape[1]\n",
        "\n",
        "    # change the shape to:\n",
        "    # (N, T, h, d_k) -> (N, h, T, d_k)\n",
        "    # in order for matrix multiply to work properly\n",
        "    q = q.view(N, T, self.n_heads, self.d_k).transpose(1, 2)\n",
        "    k = k.view(N, T, self.n_heads, self.d_k).transpose(1, 2)\n",
        "    v = v.view(N, T, self.n_heads, self.d_k).transpose(1, 2)\n",
        "\n",
        "    # compute attention weights\n",
        "    # (N, h, T, d_k) x (N, h, d_k, T) --> (N, h, T, T)\n",
        "    attn_scores = q @ k.transpose(-2, -1) / math.sqrt(self.d_k)\n",
        "    if mask is not None:\n",
        "      attn_scores = attn_scores.masked_fill(\n",
        "          mask[:, None, None, :] == 0, float('-inf'))\n",
        "    attn_weights = F.softmax(attn_scores, dim=-1)\n",
        "    \n",
        "    # compute attention-weighted values\n",
        "    # (N, h, T, T) x (N, h, T, d_k) --> (N, h, T, d_k)\n",
        "    A = attn_weights @ v\n",
        "\n",
        "    # reshape it back before final linear layer\n",
        "    A = A.transpose(1, 2) # (N, T, h, d_k)\n",
        "    A = A.contiguous().view(N, T, self.d_k * self.n_heads) # (N, T, h*d_k)\n",
        "\n",
        "    # projection\n",
        "    return self.fc(A)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class TransformerBlock(nn.Module):\n",
        "  def __init__(self, d_k, d_model, no_heads, dropout_prob=0.1):\n",
        "    super().__init__()\n",
        "\n",
        "    self.ln1 = nn.LayerNorm(d_model)\n",
        "    self.ln2 = nn.LayerNorm(d_model)\n",
        "    # You can assign submodules as regular class attributes....\n",
        "    self.mha = MultiHeadAttention(d_k, d_model, no_heads)\n",
        "    # You can stack the feed forward NN and add dropouts as many as required\n",
        "    self.ann = nn.Sequential(\n",
        "        nn.Linear(d_model, d_model * 4),\n",
        "        nn.GELU(),\n",
        "        nn.Linear(d_model * 4, d_model),\n",
        "        nn.Dropout(dropout_prob),\n",
        "    )\n",
        "    self.dropout = nn.Dropout(p=dropout_prob)\n",
        "  \n",
        "  def forward(self, x, mask=None):\n",
        "    x = self.ln1(x + self.mha(x, mask))\n",
        "    x = self.ln2(x + self.ann(x))\n",
        "    x = self.dropout(x)\n",
        "    return x"
      ],
      "metadata": {
        "id": "_MJjv-PDnV8R"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class PositionalEncoding(nn.Module):\n",
        "  def __init__(self, d_model, max_len=2048, dropout_prob=0.1):\n",
        "    super().__init__()\n",
        "    self.dropout = nn.Dropout(p=dropout_prob)\n",
        "\n",
        "    position = torch.arange(max_len).unsqueeze(1)\n",
        "    exp_term = torch.arange(0, d_model, 2)\n",
        "    div_term = torch.exp(exp_term * (-math.log(10000.0) / d_model))\n",
        "    pe = torch.zeros(1, max_len, d_model)\n",
        "    pe[0, :, 0::2] = torch.sin(position * div_term)\n",
        "    pe[0, :, 1::2] = torch.cos(position * div_term)\n",
        "    self.register_buffer('pe', pe)\n",
        "\n",
        "  def forward(self, x):\n",
        "    # x.shape: N x T x D\n",
        "    x = x + self.pe[:, :x.size(1), :]\n",
        "    return self.dropout(x)"
      ],
      "metadata": {
        "id": "IFBdQzLAnWvU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class Encoder(nn.Module):\n",
        "  def __init__(self,\n",
        "               vocab_size,\n",
        "               max_len,\n",
        "               d_k,\n",
        "               d_model,\n",
        "               no_heads,\n",
        "               n_layers,\n",
        "               n_classes,\n",
        "               dropout_prob):\n",
        "    super().__init__()\n",
        "\n",
        "    self.embedding = nn.Embedding(vocab_size, d_model)\n",
        "    self.pos_encoding = PositionalEncoding(d_model, max_len, dropout_prob)\n",
        "    transformer_blocks = [TransformerBlock(d_k, d_model, no_heads, dropout_prob) for _ in range(n_layers)]\n",
        "    self.transformer_blocks = nn.Sequential(*transformer_blocks)\n",
        "    self.ln = nn.LayerNorm(d_model)\n",
        "    self.fc = nn.Linear(d_model, n_classes)\n",
        "  \n",
        "  def forward(self, x, mask=None):\n",
        "    x = self.embedding(x)\n",
        "    x = self.pos_encoding(x)\n",
        "    for block in self.transformer_blocks:\n",
        "      x = block(x, mask)\n",
        "\n",
        "    # many-to-one (x has the shape N x T x D)\n",
        "    # Take the first T row\n",
        "    x = x[:, 0, :]\n",
        "\n",
        "    x = self.ln(x)\n",
        "    x = self.fc(x)\n",
        "    return x"
      ],
      "metadata": {
        "id": "xh_cXbVAnZGo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = Encoder(20_000, 1024, 16, 64, 4, 2, 5, 0.1)"
      ],
      "metadata": {
        "id": "wy5rokyendi3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Create device object to the GPU\n",
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "print(device)\n",
        "\n",
        "# Move model to the gpu if available\n",
        "model.to(device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cPyJqK5_nf4c",
        "outputId": "e867185d-30bf-4d6a-c148-7313ea95e7a2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "cuda:0\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Encoder(\n",
              "  (embedding): Embedding(20000, 64)\n",
              "  (pos_encoding): PositionalEncoding(\n",
              "    (dropout): Dropout(p=0.1, inplace=False)\n",
              "  )\n",
              "  (transformer_blocks): Sequential(\n",
              "    (0): TransformerBlock(\n",
              "      (ln1): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
              "      (ln2): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
              "      (mha): MultiHeadAttention(\n",
              "        (key): Linear(in_features=64, out_features=64, bias=True)\n",
              "        (query): Linear(in_features=64, out_features=64, bias=True)\n",
              "        (value): Linear(in_features=64, out_features=64, bias=True)\n",
              "        (fc): Linear(in_features=64, out_features=64, bias=True)\n",
              "      )\n",
              "      (ann): Sequential(\n",
              "        (0): Linear(in_features=64, out_features=256, bias=True)\n",
              "        (1): GELU(approximate='none')\n",
              "        (2): Linear(in_features=256, out_features=64, bias=True)\n",
              "        (3): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (1): TransformerBlock(\n",
              "      (ln1): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
              "      (ln2): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
              "      (mha): MultiHeadAttention(\n",
              "        (key): Linear(in_features=64, out_features=64, bias=True)\n",
              "        (query): Linear(in_features=64, out_features=64, bias=True)\n",
              "        (value): Linear(in_features=64, out_features=64, bias=True)\n",
              "        (fc): Linear(in_features=64, out_features=64, bias=True)\n",
              "      )\n",
              "      (ann): Sequential(\n",
              "        (0): Linear(in_features=64, out_features=256, bias=True)\n",
              "        (1): GELU(approximate='none')\n",
              "        (2): Linear(in_features=256, out_features=64, bias=True)\n",
              "        (3): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "  )\n",
              "  (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
              "  (fc): Linear(in_features=64, out_features=5, bias=True)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Generate random dataset for code testing\n",
        "# T = sequence lenght\n",
        "x = np.random.randint(0, 20_000, size=(8, 512)) # N = 8, T = 512\n",
        "x_t = torch.tensor(x).to(device)\n",
        "\n",
        "# Generate data mask\n",
        "mask = np.ones((8, 512))\n",
        "mask[:, 256:] = 0\n",
        "mask_t = torch.tensor(mask).to(device)"
      ],
      "metadata": {
        "id": "IZUUjp1pnmxc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Expected output size = N x num_classes\n",
        "y = model(x_t, mask_t)\n",
        "y.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1iIlIjiYtfib",
        "outputId": "c818642f-c000-43eb-99ab-b813b005033b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([8, 5])"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Sentiment Analysis With Above Model."
      ],
      "metadata": {
        "id": "-3WWOKkDt_gx"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Get, Preprocess and Load Data - Hugging Face. \n",
        "\n",
        "*   Use an Autotokenizer and DataCollatorWithPadding from hugging face library to tokenize data and generate padding mask.\n",
        "*   Load data with torc dataloader.\n",
        "\n"
      ],
      "metadata": {
        "id": "6EHkLf6a_gdJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "! pip install transformers datasets"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "w7EjLrPd_u-F",
        "outputId": "55ecbfb1-2394-49f4-b652-923fce152929"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.28.1-py3-none-any.whl (7.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.0/7.0 MB\u001b[0m \u001b[31m70.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting datasets\n",
            "  Downloading datasets-2.11.0-py3-none-any.whl (468 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m468.7/468.7 kB\u001b[0m \u001b[31m43.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (2022.10.31)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from transformers) (2.27.1)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.9/dist-packages (from transformers) (4.65.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from transformers) (3.11.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (1.22.4)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from transformers) (6.0)\n",
            "Collecting huggingface-hub<1.0,>=0.11.0\n",
            "  Downloading huggingface_hub-0.14.1-py3-none-any.whl (224 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m224.5/224.5 kB\u001b[0m \u001b[31m28.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
            "  Downloading tokenizers-0.13.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m106.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/dist-packages (from transformers) (23.1)\n",
            "Collecting dill<0.3.7,>=0.3.0\n",
            "  Downloading dill-0.3.6-py3-none-any.whl (110 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m110.5/110.5 kB\u001b[0m \u001b[31m15.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting multiprocess\n",
            "  Downloading multiprocess-0.70.14-py39-none-any.whl (132 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m132.9/132.9 kB\u001b[0m \u001b[31m17.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.9/dist-packages (from datasets) (2023.4.0)\n",
            "Collecting responses<0.19\n",
            "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
            "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (9.0.0)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.9/dist-packages (from datasets) (1.5.3)\n",
            "Collecting xxhash\n",
            "  Downloading xxhash-3.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.2/212.2 kB\u001b[0m \u001b[31m26.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting aiohttp\n",
            "  Downloading aiohttp-3.8.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m75.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (23.1.0)\n",
            "Collecting async-timeout<5.0,>=4.0.0a3\n",
            "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
            "Collecting frozenlist>=1.1.1\n",
            "  Downloading frozenlist-1.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (158 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m158.8/158.8 kB\u001b[0m \u001b[31m23.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting yarl<2.0,>=1.0\n",
            "  Downloading yarl-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (269 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m269.4/269.4 kB\u001b[0m \u001b[31m33.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (2.0.12)\n",
            "Collecting multidict<7.0,>=4.5\n",
            "  Downloading multidict-6.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.2/114.2 kB\u001b[0m \u001b[31m15.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting aiosignal>=1.1.2\n",
            "  Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (4.5.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (2022.12.7)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (3.4)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.9/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.9/dist-packages (from pandas->datasets) (2022.7.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.9/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n",
            "Installing collected packages: tokenizers, xxhash, multidict, frozenlist, dill, async-timeout, yarl, responses, multiprocess, huggingface-hub, aiosignal, transformers, aiohttp, datasets\n",
            "Successfully installed aiohttp-3.8.4 aiosignal-1.3.1 async-timeout-4.0.2 datasets-2.11.0 dill-0.3.6 frozenlist-1.3.3 huggingface-hub-0.14.1 multidict-6.0.4 multiprocess-0.70.14 responses-0.18.0 tokenizers-0.13.3 transformers-4.28.1 xxhash-3.2.0 yarl-1.9.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoTokenizer, DataCollatorWithPadding"
      ],
      "metadata": {
        "id": "b_Mifpi-ATyP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained('distilbert-base-cased')\n",
        "tokenizer"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 181,
          "referenced_widgets": [
            "261ba7d1591d4c0cb9ba4a0c693689d1",
            "78170014c2f9401fb0423d7386016a43",
            "3b2a6311492f44808c1b98fd2a6267f0",
            "09e9e116fedf482788c38ee3b31f8181",
            "8132cf3198e04ef0923a10b69b50b482",
            "f2d760da92b94e29b0fda376f4fc2cc0",
            "5ecad9dbd3a04e09ba6905e907a30f69",
            "192ff591db244ff188c83fab5af77025",
            "32363a859ea9436abd98d6f98785fc77",
            "53a41a265a1747b6b6e918b7d24a665e",
            "eedbd1c4a8f9400986868e92152ed430",
            "20d969774f8c4a5fa849a016c8d4e900",
            "7a9b8f0b909a4f9699004c68714db553",
            "683cb16663ca4a58bb63ffd9131c8710",
            "1108b929352c4edfa777a624b6dda618",
            "86f5827603fa4651baec9bd4e064a6cb",
            "81f343be7e954b0287d5d7870a24973e",
            "031974c2d2a24dffb0a43f89c7761349",
            "f2b1d14879f5493baec35cf4e05c8ac2",
            "2a185b5b038341b6ab34dc30c96b7cd5",
            "2db910cbb5d6400bbcdabd8d404e98a0",
            "72b1d461ba2640f1b79c8f41433e66a6",
            "3a6a7ac5fb174a19b340b5dbe7fead8c",
            "048f56ca151244d88c5f6d5c7b780a6f",
            "443282a1b1c14ddf9205740cebe80cc0",
            "2aa47f3440804744aeb84395f4dff92a",
            "43bd3e8bf71a4d15b23dfebd60e553fc",
            "4514cd18db37473d81e4a82765cb632b",
            "3c9c1bc50d464bd895a2b4ff74c1973c",
            "64db571b882a4f298323d361f9352c19",
            "bb169d2ef43a4cfeaf90bb3c7aa2a375",
            "f0a99381214740339d8c90198c58130f",
            "c9669d95d5584fb584772360943a101d",
            "253daa13e88c49b399baba35264ca402",
            "0d1a852a8f854a44be114224c6c4c062",
            "d7c3404d9b7f419b82f32586a27e4a83",
            "08ee9f12fdcd4e4fb7738e6c4fa75210",
            "b34f99299cfb4a56b1384d2bbeee0ec1",
            "d202856cb9604263b20f8cd6d08895c8",
            "449ef33ac6e441c6aab3c853722aec05",
            "ae808eb0d3a84f55b6b31e1d20e2a507",
            "f41489f07ce3417d8d911f292470230a",
            "e4e3e954bb1649448dd4dc4dc63a6772",
            "e3eb25432dfe4a009e393282338916fa"
          ]
        },
        "id": "fMUutK5SAamc",
        "outputId": "49825ce6-7e16-4cd9-d20b-4852fb35e69f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)okenizer_config.json:   0%|          | 0.00/29.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "261ba7d1591d4c0cb9ba4a0c693689d1"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/411 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "20d969774f8c4a5fa849a016c8d4e900"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/213k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3a6a7ac5fb174a19b340b5dbe7fead8c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)/main/tokenizer.json:   0%|          | 0.00/436k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "253daa13e88c49b399baba35264ca402"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DistilBertTokenizerFast(name_or_path='distilbert-base-cased', vocab_size=28996, model_max_length=512, is_fast=True, padding_side='right', truncation_side='right', special_tokens={'unk_token': '[UNK]', 'sep_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]', 'mask_token': '[MASK]'}, clean_up_tokenization_spaces=True)"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import load_dataset\n",
        "raw_datasets = load_dataset(\"glue\", \"sst2\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 233,
          "referenced_widgets": [
            "aead216a8d1d475a81e1e5da25630529",
            "83a36ead7d704204a26234b0e0f7ad65",
            "013f63f22c684c8c8bb8895ce3d1b2e0",
            "7c1adde1d85a4479a6c3941063310488",
            "1423aae4c2c448cd83c8e2091a092218",
            "1883467b5c1248e8afc125e8721744d1",
            "3d258350145c43d6bdfc25f157e8c709",
            "c99e9f56c9a24b68a764cf85e9edc302",
            "c503e1a20bb24c058777ea2f5ed33652",
            "db81134a1aec44acb03500ca4543f8e1",
            "2217294ce415481e8363b9d02cf48e32",
            "9f14a0a8a3944d8f8e941f5202f84c68",
            "2d6f09cd27d24131b7b5862a9a8e3915",
            "732bff4ae2814025aa8e5ad808f6c982",
            "606be339b67741ebae6c844fae69de33",
            "e0ebff0f593f4dfd98a5a22a58129ed2",
            "975a171470bd4fb2b93d911f64b68321",
            "fa5bf5ad234347768160daaab01f64d5",
            "8b9ab61c150044e1b42df799bc03e5dd",
            "10cc83a92ce347ac9d018f8d57d81fc1",
            "ba0996fd89ea4b55910f1c466be2ec87",
            "041fe4e0e35244ee824c550bdbd2be82",
            "c0e9c152b79d443c92fbd419e5287032",
            "640808c3ac784381a45a949f082380e3",
            "b25f03bbb46249f6b1e00eacbd263bca",
            "114972f375bd40e8bb70a1a0545999e5",
            "29c66432e32345638ff08bb1bca40889",
            "75cd7ec346a84ccc97d0b5fe8dcfc2be",
            "5c6becc1712749bb97b1de48aa87d42f",
            "cad092cef9db46a88ba5433b625fb4d3",
            "f68f9db39ac8475290e5da7237ba93df",
            "0893c31ac23e4ab38f1357100c20cba3",
            "9f772aafa22e4ac7b8975ec4c249ec84",
            "36fbc27c01d0450586072e96ec1df67e",
            "9579693b195545e9988118fd3cc6f66e",
            "fc65da9374da43cf9e247d487e2f183a",
            "338a93a4ed1744ca984969f2dc8ecc0c",
            "5e6bfcf8cbcf4bf091d6b0f41ed98076",
            "d696f191259e462ba846b5c7d114dc19",
            "94cee836e00c470eaff060582dd6a805",
            "289ad15f9d9e4e61afa95a55709482ac",
            "e9e6bfeae8254189b8517e22c6d1bdd4",
            "da0954563236433db5002d6a8b7b65ff",
            "dfe2b1130d134d9da9ada1e5a6daf4a2",
            "b36db7a7342d4c2ab63bdb7c7a8eb586",
            "33b8ad2509c949a5858899a65277e6d6",
            "d0b47698ce0c481a9e00549160454938",
            "9929ea473808449e8c50149afbffb3a9",
            "d7d3fb82f9284cb3a44778be0c7bddf3",
            "7879d0a1b30746d88a182579da17afa7",
            "966c72e7e9f24b2abf2beef693c58915",
            "1bd0a41737c749ac8062a2c7e30d4283",
            "63755d77b99d4f809ecc0420889df29a",
            "5bdcd4f164644d9293cf90c43d5d0568",
            "9aadce43516f4100a83d8a908892859a",
            "d81a9ddbf4f44c0dbf4dfc9a0aa84f70",
            "b648b7c85f3041e297125b408f3a09f5",
            "414b053a920b403080116cca0cafdedf",
            "90a9e4398b8443fe835044c1fd9168b9",
            "ee793a41dedb4c77a31745f0f7cdd5d2",
            "f2f4686760374c7796b21d6ac5430110",
            "904f722c8be0491cbdfe5ed859c93fa8",
            "549942a03c1b47c0aac80bad1f8ffee0",
            "76947130098447c48631d3580375a2e9",
            "fc7b52d0fe1b4ed787c0d1c8c3ea87ed",
            "fa20464943ce41d2baa08420fa833f10",
            "0ff204cac2b2450cb4b25111125b4b05",
            "cabf04191cb748a6b391d017b05d2a47",
            "406a22231bf24667baec2c5f66bd092e",
            "0ffba5428c13451c80469591cf051e9f",
            "d70e886d78a24d4f9574673020e28bdc",
            "6eea52594be141c6a770ae5d29a92701",
            "bc2ef94e2c4f43aeae42650345026664",
            "2c5f085bfbce486aa2f94a04deb675d7",
            "05dc68e3c85842549f919f2dfbd00ee6",
            "19807e5f36a948e0a8de927a20ad4e97",
            "74fac346ddbb482190c66450c146c1ef",
            "3398c91da17c472793574a3c86c63f0e",
            "21713f2bd5834571a23b2ca87626c34d",
            "2313a9d7236b4dcba4ae206aa85d90fa",
            "0072c675564048ddaa887caf8f509025",
            "0e169378ce6a41ba86b04f4534531c68",
            "ad041993bd234b9d894b3b66db224788",
            "787a1414d9274b8cbaf97f351a626e98",
            "0a9c7ad3f5544510aff1db42a2291b12",
            "f6769f2d24d249f3b3f63ca6fce0c1b8",
            "8f6078ac6ba04302bfced59497be94c0",
            "b0e692e508844336ba43a22dc3eda1e8"
          ]
        },
        "id": "KEc0wV6uA4Oi",
        "outputId": "b9a19139-4896-4288-a076-1ad7eecbb8a3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading builder script:   0%|          | 0.00/28.8k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "aead216a8d1d475a81e1e5da25630529"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading metadata:   0%|          | 0.00/28.7k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9f14a0a8a3944d8f8e941f5202f84c68"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading readme:   0%|          | 0.00/27.9k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c0e9c152b79d443c92fbd419e5287032"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading and preparing dataset glue/sst2 to /root/.cache/huggingface/datasets/glue/sst2/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad...\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data:   0%|          | 0.00/7.44M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "36fbc27c01d0450586072e96ec1df67e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating train split:   0%|          | 0/67349 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b36db7a7342d4c2ab63bdb7c7a8eb586"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating validation split:   0%|          | 0/872 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d81a9ddbf4f44c0dbf4dfc9a0aa84f70"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating test split:   0%|          | 0/1821 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0ff204cac2b2450cb4b25111125b4b05"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Dataset glue downloaded and prepared to /root/.cache/huggingface/datasets/glue/sst2/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad. Subsequent calls will reuse this data.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/3 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3398c91da17c472793574a3c86c63f0e"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "raw_datasets"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QRju2mS-AvIk",
        "outputId": "738f675c-cf92-4c57-f0c8-6cfaac8eadb8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['sentence', 'label', 'idx'],\n",
              "        num_rows: 67349\n",
              "    })\n",
              "    validation: Dataset({\n",
              "        features: ['sentence', 'label', 'idx'],\n",
              "        num_rows: 872\n",
              "    })\n",
              "    test: Dataset({\n",
              "        features: ['sentence', 'label', 'idx'],\n",
              "        num_rows: 1821\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def tokenize_fn(batch):\n",
        "  return tokenizer(batch['sentence'], truncation=True)\n",
        "# map toeknize function to dataset\n",
        "tokenized_datasets = raw_datasets.map(tokenize_fn, batched=True)\n",
        "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "1b42bba5ee964eb29b7985c7a2713a77",
            "66e84e5b7ab44557a26ecad20ad7f570",
            "fb2a0b35683b44ee8ed9b255f1342398",
            "0350852ab4bc417e9da41665b6289607",
            "5483f91b5780410dbe5f60ecd15a99bd",
            "f34b5f53d83e4e1dadbcde31d507b9cf",
            "721242311de741498468f0d5d1dfdaa5",
            "47e46aaeb22a462d86e6af0076f7c3af",
            "a952f3b9073648828113d2f81af77ccd",
            "737b9108dae64e0aae2086b835289bfb",
            "d376e9b943d5458a87a0005aa88a702d",
            "787574f727d5484889bbc298b250644d",
            "27bace151ccf4dcdbeb85474f9dc2edc",
            "00c1cadd1a7d4581a9a62995576d0b18",
            "53a83096e7a44570a42734f34c25f223",
            "cac44d6cb58e46cb9972afbaf699f5e1",
            "5dbbe6b3557e4fc19079ca8412a35317",
            "6d271f007cb445ec8e77c4e53a52e2dd",
            "2ccd841be81e4c2e97aec55c5dc33e1a",
            "cd210bd9760c48b08785d227083d710f",
            "a85c5c4270f8442abff4c2a80941742f",
            "0257f4602f5f47ab9e20a82cfa04bcb1",
            "bd395ab50f5f41c3b379e1d700380a90",
            "7424f8a3cf6a43558d87981538132a5d",
            "a36ca74068464eae938c62989ae50c18",
            "35feb2c81b08484d98c5233a90878864",
            "c963602d1b804aae82ab2091c9315399",
            "e662511e959b41be8f039703a9f240c7",
            "5a8f9417648e4d60bbec52f0e07ae696",
            "a7b0d43005fa4428b5c60d699e29823d",
            "ccb1066309bf4f66a947bdb2009163b0",
            "2cabf3d925e948eb9d7ef001e3c67ee6",
            "98f8ebbbb59340f0adfb89757d3b071c"
          ]
        },
        "id": "cprLJc-MEpzq",
        "outputId": "726cb8c0-4319-4412-9fd9-4bb021b9763c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/67349 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "1b42bba5ee964eb29b7985c7a2713a77"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/872 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "787574f727d5484889bbc298b250644d"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/1821 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "bd395ab50f5f41c3b379e1d700380a90"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized_datasets"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "etsgx1q7FMrL",
        "outputId": "979e5500-4bf5-46dc-8be0-cd45a97b14f4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['sentence', 'label', 'idx', 'input_ids', 'attention_mask'],\n",
              "        num_rows: 67349\n",
              "    })\n",
              "    validation: Dataset({\n",
              "        features: ['sentence', 'label', 'idx', 'input_ids', 'attention_mask'],\n",
              "        num_rows: 872\n",
              "    })\n",
              "    test: Dataset({\n",
              "        features: ['sentence', 'label', 'idx', 'input_ids', 'attention_mask'],\n",
              "        num_rows: 1821\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized_datasets = tokenized_datasets.remove_columns([\"sentence\", \"idx\"])\n",
        "tokenized_datasets = tokenized_datasets.rename_column(\"label\", \"labels\")"
      ],
      "metadata": {
        "id": "0-Z3eMMbFPtf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized_datasets"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uPxr9KsgHRPr",
        "outputId": "f84f570e-134a-4511-8805-8609d3712e7a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['labels', 'input_ids', 'attention_mask'],\n",
              "        num_rows: 67349\n",
              "    })\n",
              "    validation: Dataset({\n",
              "        features: ['labels', 'input_ids', 'attention_mask'],\n",
              "        num_rows: 872\n",
              "    })\n",
              "    test: Dataset({\n",
              "        features: ['labels', 'input_ids', 'attention_mask'],\n",
              "        num_rows: 1821\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Load datasets."
      ],
      "metadata": {
        "id": "37HW3ov7F6db"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from torch.utils.data import DataLoader\n",
        "\n",
        "train_loader = DataLoader(\n",
        "    tokenized_datasets[\"train\"],\n",
        "    shuffle=True,\n",
        "    batch_size=32,\n",
        "    collate_fn=data_collator\n",
        ")\n",
        "valid_loader = DataLoader(\n",
        "    tokenized_datasets[\"validation\"],\n",
        "    batch_size=32,\n",
        "    collate_fn=data_collator\n",
        ")"
      ],
      "metadata": {
        "id": "VM7lNfTxF5x4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# check how it works\n",
        "for batch in train_loader:\n",
        "  for k, v in batch.items():\n",
        "    print(\"k:\", k, \"v.shape:\", v.shape)\n",
        "  break"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XA9UOxaRG0KG",
        "outputId": "ab99f44d-ede6-4bc1-a6f4-eb61e93626eb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "You're using a DistilBertTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "k: labels v.shape: torch.Size([32])\n",
            "k: input_ids v.shape: torch.Size([32, 45])\n",
            "k: attention_mask v.shape: torch.Size([32, 45])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "batch.items"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cqF9gFGAG22p",
        "outputId": "7891a89b-f016-4025-faa8-e3fec631d9ed"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<bound method BatchEncoding.items of {'labels': tensor([0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 0, 1, 0, 1,\n",
              "        1, 1, 1, 0, 0, 1, 1, 0]), 'input_ids': tensor([[  101,   179, 15818,  ...,     0,     0,     0],\n",
              "        [  101,  1110,   170,  ...,     0,     0,     0],\n",
              "        [  101,  1106,  3994,  ...,     0,     0,     0],\n",
              "        ...,\n",
              "        [  101,  1128,  1431,  ...,     0,     0,     0],\n",
              "        [  101,  1122,   112,  ...,     0,     0,     0],\n",
              "        [  101,  1104,  1103,  ...,     0,     0,     0]]), 'attention_mask': tensor([[1, 1, 1,  ..., 0, 0, 0],\n",
              "        [1, 1, 1,  ..., 0, 0, 0],\n",
              "        [1, 1, 1,  ..., 0, 0, 0],\n",
              "        ...,\n",
              "        [1, 1, 1,  ..., 0, 0, 0],\n",
              "        [1, 1, 1,  ..., 0, 0, 0],\n",
              "        [1, 1, 1,  ..., 0, 0, 0]])}>"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Encoder Model"
      ],
      "metadata": {
        "id": "kHzAEU5xHteb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "checkpoint = 'distilbert-base-cased'\n",
        "model = Encoder(\n",
        "    vocab_size=tokenizer.vocab_size,\n",
        "    max_len=tokenizer.max_model_input_sizes[checkpoint],\n",
        "    d_k=16,\n",
        "    d_model=64,\n",
        "    no_heads=4,\n",
        "    n_layers=2,\n",
        "    n_classes=2,\n",
        "    dropout_prob=0.1,\n",
        ")\n",
        "model.to(device)"
      ],
      "metadata": {
        "id": "1HuNYk1YG4eh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Instantiate Model Loss, Optimizer(s) and train."
      ],
      "metadata": {
        "id": "ynBLL_5eIPtK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "criterion = nn.CrossEntropyLoss()\n",
        "optimizer = torch.optim.Adam(model.parameters())"
      ],
      "metadata": {
        "id": "hQ45vDRSIWKg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from datetime import datetime\n",
        "# Training loop\n",
        "def train(model, criterion, optimizer, train_loader, valid_loader, epochs):\n",
        "  train_losses = np.zeros(epochs)\n",
        "  test_losses = np.zeros(epochs)\n",
        "\n",
        "  for it in range(epochs):\n",
        "    model.train()   # Model in training mode\n",
        "    t0 = datetime.now()\n",
        "    train_loss = 0\n",
        "    n_train = 0\n",
        "    for batch in train_loader:\n",
        "      # move data to GPU\n",
        "      batch = {k: v.to(device) for k, v in batch.items()}\n",
        "\n",
        "      # zero the parameter gradients\n",
        "      optimizer.zero_grad()\n",
        "\n",
        "      # Forward pass\n",
        "      outputs = model(batch['input_ids'], batch['attention_mask'])\n",
        "      loss = criterion(outputs, batch['labels'])\n",
        "        \n",
        "      # Backward and optimize\n",
        "      loss.backward()   # Compute Gradients (Back prop)\n",
        "      optimizer.step()  # Update weights(GD/Adam)\n",
        "\n",
        "      train_loss += loss.item()*batch['input_ids'].size(0)\n",
        "      n_train += batch['input_ids'].size(0)\n",
        "\n",
        "    # Get average train loss\n",
        "    train_loss = train_loss / n_train\n",
        "    \n",
        "    # Evalaute model at the end of each epoch\n",
        "    model.eval()\n",
        "    test_loss = 0\n",
        "    n_test = 0\n",
        "    for batch in valid_loader:\n",
        "      batch = {k: v.to(device) for k, v in batch.items()}\n",
        "      outputs = model(batch['input_ids'], batch['attention_mask'])\n",
        "      loss = criterion(outputs, batch['labels'])\n",
        "      test_loss += loss.item()*batch['input_ids'].size(0)\n",
        "      n_test += batch['input_ids'].size(0)\n",
        "    test_loss = test_loss / n_test\n",
        "\n",
        "    # Save losses\n",
        "    train_losses[it] = train_loss\n",
        "    test_losses[it] = test_loss\n",
        "    \n",
        "    dt = datetime.now() - t0\n",
        "    print(f'Epoch {it+1}/{epochs}, Train Loss: {train_loss:.4f}, \\\n",
        "      Test Loss: {test_loss:.4f}, Duration: {dt}')\n",
        "  \n",
        "  return train_losses, test_losses"
      ],
      "metadata": {
        "id": "rzDfyenmJM1V"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now Train"
      ],
      "metadata": {
        "id": "0vFhKiR1SIhD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "train(model, criterion, optimizer, train_loader, valid_loader, epochs=4)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Krg5LnUsSJfE",
        "outputId": "7ff58267-0b5d-4e1b-f430-a4b70a667f6c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/4, Train Loss: 0.4973,       Test Loss: 0.5238, Duration: 0:00:20.227275\n",
            "Epoch 2/4, Train Loss: 0.3547,       Test Loss: 0.5192, Duration: 0:00:17.718122\n",
            "Epoch 3/4, Train Loss: 0.2958,       Test Loss: 0.5052, Duration: 0:00:18.772279\n",
            "Epoch 4/4, Train Loss: 0.2567,       Test Loss: 0.5356, Duration: 0:00:18.641025\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array([0.49727873, 0.35472264, 0.29578105, 0.25674538]),\n",
              " array([0.52382502, 0.51919762, 0.50519833, 0.53563062]))"
            ]
          },
          "metadata": {},
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Accuracy Evaluation"
      ],
      "metadata": {
        "id": "pURC1G-iTRiK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Accuracy\n",
        "model.eval()\n",
        "n_correct = 0.\n",
        "n_total = 0.\n",
        "for batch in train_loader:\n",
        "  # Move to GPU\n",
        "  batch = {k: v.to(device) for k, v in batch.items()}\n",
        "\n",
        "  # Forward pass\n",
        "  outputs = model(batch['input_ids'], batch['attention_mask'])\n",
        "\n",
        "  # Get prediction\n",
        "  # torch.max returns both max and argmax\n",
        "  _, predictions = torch.max(outputs, 1)\n",
        "  \n",
        "  # update counts\n",
        "  n_correct += (predictions == batch['labels']).sum().item()\n",
        "  n_total += batch['labels'].shape[0]\n",
        "\n",
        "train_acc = n_correct / n_total\n",
        "\n",
        "\n",
        "n_correct = 0.\n",
        "n_total = 0.\n",
        "for batch in valid_loader:\n",
        "  # Move to GPU\n",
        "  batch = {k: v.to(device) for k, v in batch.items()}\n",
        "  \n",
        "  # Forward pass\n",
        "  outputs = model(batch['input_ids'], batch['attention_mask'])\n",
        "\n",
        "  # Get prediction\n",
        "  # torch.max returns both max and argmax\n",
        "  _, predictions = torch.max(outputs, 1)\n",
        "  \n",
        "  # update counts\n",
        "  n_correct += (predictions == batch['labels']).sum().item()\n",
        "  n_total += batch['labels'].shape[0]\n",
        "\n",
        "test_acc = n_correct / n_total\n",
        "print(f\"Train acc: {train_acc:.4f}, Test acc: {test_acc:.4f}\")"
      ],
      "metadata": {
        "id": "QV8QHWf1TItN",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a7a4e6d5-5936-47f0-ea72-c25831fe800b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train acc: 0.9362, Test acc: 0.7959\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Time Series Data Prediction With Above Model."
      ],
      "metadata": {
        "id": "mzV0sW5zV78R"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Modify Encoder For Time series data and Regression task. Take note --\n",
        "\n",
        "\n",
        "*   Mask = None, because the sequence lenght does not change given the way the data was prepared.\n",
        "*   No embedding layer is required and the positional encoding will be added to x inputs directly.\n",
        "\n",
        "\n",
        "*   max_lenght = T\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "rtQIMLuFWXTm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "T = 5    # Also max_lenght\n",
        "D = 8    # Also d_model\n",
        "K = 1    # Also num_classes"
      ],
      "metadata": {
        "id": "h5tkHPMxa7Vq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fHRsYMCLuAVw"
      },
      "outputs": [],
      "source": [
        "# Excerpt form lazy programmer NLP course....\n",
        "# Subclass the torch.nn class\n",
        "class MultiHeadAttention(nn.Module):\n",
        "  def __init__(self, d_k, d_model, no_heads):\n",
        "    super().__init__()\n",
        "    # d_k = key dimension, d_q = query dimension and d_v = value dimension\n",
        "    # d_k and d_q have the same dimension, d_v is also assumed to be d_k\n",
        "    self.d_k = d_k\n",
        "    self.n_heads = no_heads\n",
        "\n",
        "    # input feature size = size of x = (N,T,d_model)\n",
        "    # multi-head vectorized implementation \n",
        "    self.key = nn.Linear(d_model, d_k * no_heads)\n",
        "    self.query = nn.Linear(d_model, d_k * no_heads)\n",
        "    self.value = nn.Linear(d_model, d_k * no_heads)\n",
        "\n",
        "    # final linear layer - To convert N x T x d_v to N x T x d_model\n",
        "    self.fc = nn.Linear(d_k * no_heads, d_model)\n",
        "\n",
        "  def forward(self, x, mask=None):\n",
        "    q = self.query(x) # N x T x (h*d_k)\n",
        "    k = self.key(x)   # N x T x (h*d_k)\n",
        "    v = self.value(x) # N x T x (h*d_v)\n",
        "\n",
        "    N = q.shape[0]\n",
        "    T = q.shape[1]\n",
        "\n",
        "    # change the shape to:\n",
        "    # (N, T, h, d_k) -> (N, h, T, d_k)\n",
        "    # in order for matrix multiply to work properly\n",
        "    q = q.view(N, T, self.n_heads, self.d_k).transpose(1, 2)\n",
        "    k = k.view(N, T, self.n_heads, self.d_k).transpose(1, 2)\n",
        "    v = v.view(N, T, self.n_heads, self.d_k).transpose(1, 2)\n",
        "\n",
        "    # compute attention weights\n",
        "    # (N, h, T, d_k) x (N, h, d_k, T) --> (N, h, T, T)\n",
        "    attn_scores = q @ k.transpose(-2, -1) / math.sqrt(self.d_k)\n",
        "    if mask is not None:\n",
        "      attn_scores = attn_scores.masked_fill(\n",
        "          mask[:, None, None, :] == 0, float('-inf'))\n",
        "    attn_weights = F.softmax(attn_scores, dim=-1)\n",
        "    \n",
        "    # compute attention-weighted values\n",
        "    # (N, h, T, T) x (N, h, T, d_k) --> (N, h, T, d_k)\n",
        "    A = attn_weights @ v\n",
        "\n",
        "    # reshape it back before final linear layer\n",
        "    A = A.transpose(1, 2) # (N, T, h, d_k)\n",
        "    A = A.contiguous().view(N, T, self.d_k * self.n_heads) # (N, T, h*d_k)\n",
        "\n",
        "    # projection\n",
        "    return self.fc(A)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Returns NxTxD_model\n",
        "model = MultiHeadAttention(4,8,3)\n",
        "model.to(device)\n",
        "y = model(x_t)\n",
        "y.shape"
      ],
      "metadata": {
        "id": "wfRS8raGts6Q",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2c1cb964-9325-4c2e-d1a1-3a1d5d3244e5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([41452, 5, 8])"
            ]
          },
          "metadata": {},
          "execution_count": 74
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class TransformerBlock(nn.Module):\n",
        "  def __init__(self, d_k, d_model, no_heads, dropout_prob=0.1):\n",
        "    super().__init__()\n",
        "\n",
        "    self.ln1 = nn.LayerNorm(d_model)\n",
        "    self.ln2 = nn.LayerNorm(d_model)\n",
        "    # You can assign submodules as regular class attributes....\n",
        "    self.mha = MultiHeadAttention(d_k, d_model, no_heads)\n",
        "    # You can stack the feed forward NN and add dropouts as many as required\n",
        "    self.ann = nn.Sequential(\n",
        "        nn.Linear(d_model, d_model * 4),\n",
        "        nn.GELU(),\n",
        "        nn.Dropout(dropout_prob),\n",
        "        nn.Linear(d_model * 4, d_model),\n",
        "        nn.Dropout(dropout_prob),\n",
        "    )\n",
        "    self.dropout = nn.Dropout(p=dropout_prob)\n",
        "  \n",
        "  def forward(self, x, mask=None):\n",
        "    x = self.ln1(x + self.mha(x, mask))\n",
        "    x = self.ln2(x + self.ann(x))\n",
        "    x = self.dropout(x)\n",
        "    return x"
      ],
      "metadata": {
        "id": "iaW4fzo7YjqB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Return NxTxD_model\n",
        "model = TransformerBlock(4, 8, 3, 0.2)\n",
        "model.to(device)\n",
        "y = model(x_t)\n",
        "y.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gDzawQAutkw2",
        "outputId": "b04d14a1-b751-459f-fa52-3ca744864549"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([41452, 5, 8])"
            ]
          },
          "metadata": {},
          "execution_count": 81
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class PositionalEncoding(nn.Module):\n",
        "  def __init__(self, d_model, max_len, dropout_prob):\n",
        "    super().__init__()\n",
        "    self.dropout = nn.Dropout(p=dropout_prob)\n",
        "\n",
        "    position = torch.arange(max_len).unsqueeze(1)\n",
        "    exp_term = torch.arange(0, d_model, 2)\n",
        "    div_term = torch.exp(exp_term * (-math.log(10000.0) / d_model))\n",
        "    pe = torch.zeros(1, max_len, d_model)\n",
        "    pe[0, :, 0::2] = torch.sin(position * div_term)\n",
        "    pe[0, :, 1::2] = torch.cos(position * div_term)\n",
        "    self.register_buffer('pe', pe)\n",
        "\n",
        "  def forward(self, x):\n",
        "    # x.shape: N x T x D\n",
        "    x = x + self.pe[:, :x.size(1), :]\n",
        "    return self.dropout(x)"
      ],
      "metadata": {
        "id": "z7t8n3GFYjqC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = PositionalEncoding(8,5,0.2)\n",
        "model.to(device)\n",
        "y = model(x_t)\n",
        "y.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6GSfxnEZsm1H",
        "outputId": "656e43b5-22ed-4150-ac95-5fab36256730"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([41452, 5, 8])"
            ]
          },
          "metadata": {},
          "execution_count": 60
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class Encoder(nn.Module):\n",
        "  def __init__(self,\n",
        "               max_len,\n",
        "               d_k,\n",
        "               d_model,\n",
        "               no_heads,\n",
        "               n_layers,\n",
        "               dropout_prob):\n",
        "    super().__init__()\n",
        "\n",
        "    self.pos_encoding = PositionalEncoding(d_model, max_len, dropout_prob)\n",
        "    transformer_blocks = [TransformerBlock(d_k, d_model, no_heads, dropout_prob) for _ in range(n_layers)]\n",
        "    self.transformer_blocks = nn.Sequential(*transformer_blocks)\n",
        "    self.ln = nn.LayerNorm(d_model)\n",
        "    self.fc = nn.Linear(d_model, 1)\n",
        "  \n",
        "  def forward(self, x, mask=None):\n",
        "    x = self.pos_encoding(x)\n",
        "    for block in self.transformer_blocks:\n",
        "      x = block(x, mask)\n",
        "\n",
        "    # many-to-one (NxTx1 --> Nx1)\n",
        "    # Take the first T row\n",
        "    x = x[:, 0, :]\n",
        "\n",
        "    x = self.ln(x)\n",
        "    x = self.fc(x)\n",
        "    return x"
      ],
      "metadata": {
        "id": "NmR8XvGZWUhN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = Encoder(5, 2, 8, 2, 3, 0.4)"
      ],
      "metadata": {
        "id": "mcyPAAY5WUhN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Create device object to the GPU\n",
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "print(device)\n",
        "\n",
        "# Move model to the gpu if available\n",
        "model.to(device)"
      ],
      "metadata": {
        "id": "bl78dFOlWUhN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Load Data "
      ],
      "metadata": {
        "id": "BtRc1C5YfHiO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import files\n",
        "uploaded = files.upload()"
      ],
      "metadata": {
        "id": "SyEdrqlI90r0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "NT_y_train = np.load('/content/NT-Y-train (1).npy')\n",
        "NT_y_val = np.load('/content/NT-Y-val (1).npy')"
      ],
      "metadata": {
        "id": "KJaG2octdh7F"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "NTD_x_train = np.load('/content/NTD-X-train.npy')\n",
        "NTD_x_val = np.load('/content/NTD-X-val.npy')"
      ],
      "metadata": {
        "id": "fFwHYAstweLs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Test with time series data of shape NxTxD"
      ],
      "metadata": {
        "id": "YlhTF_6tcNYo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# model.fc1.weight.dtype"
      ],
      "metadata": {
        "id": "7pB6LesXqqbC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(NTD_x_train.shape)\n",
        "x_t = torch.tensor(NTD_x_train).type('torch.FloatTensor').to(device)\n",
        "print(x_t.shape, type(x_t))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a3KMKgfRcH2F",
        "outputId": "a8f567e3-5e4e-46e6-dd01-c10bce0aaac1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(41452, 5, 8)\n",
            "torch.Size([41452, 5, 8]) <class 'torch.Tensor'>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Expect an output of NxK = 41452,1\n",
        "y = model(x_t)\n",
        "y.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K3OGdipPkGZh",
        "outputId": "666b56d5-0aee-454f-bc65-ca56ee386c0b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([41452, 1])"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now Let's Train and Use Torch dataloader to batch data"
      ],
      "metadata": {
        "id": "UQyyF2TLyRYQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "NT_y_train.reshape(-1,1).shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XwtDnSFArtiT",
        "outputId": "bc8ddb12-d652-41c6-d360-267faea4fc48"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(41452, 1)"
            ]
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x_train = torch.tensor(NTD_x_train).type('torch.FloatTensor')\n",
        "x_val = torch.tensor(NTD_x_val).type('torch.FloatTensor')\n",
        "y_train = torch.tensor(NT_y_train.reshape(-1,1)).type('torch.FloatTensor')\n",
        "y_val = torch.tensor(NT_y_val.reshape(-1,1)).type('torch.FloatTensor')"
      ],
      "metadata": {
        "id": "17U-xVjKzdkM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from torch.utils.data import TensorDataset\n",
        "train_dataset = TensorDataset(x_train, y_train)\n",
        "val_dataset = TensorDataset(x_val, y_val)"
      ],
      "metadata": {
        "id": "ySn_MHo-0gpB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from torch.utils.data import DataLoader\n",
        "train_loader = DataLoader(train_dataset,\n",
        "    batch_size=32)\n",
        "\n",
        "valid_loader = DataLoader(val_dataset,\n",
        "    batch_size=32)"
      ],
      "metadata": {
        "id": "YAsu7RQUymGg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# check how it works\n",
        "for batch in train_loader:\n",
        "  k=batch[0]\n",
        "  v=batch[1]\n",
        "  print(\"k:\", k.shape, \"v.shape:\", v.shape)\n",
        "  break"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3C2jA0AO1Iiu",
        "outputId": "2a4a431d-02ff-454f-e21b-5012339aeec2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "k: torch.Size([32, 5, 8]) v.shape: torch.Size([32, 1])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "batch[0].shape[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0SVHN4A7sWkL",
        "outputId": "66566ad8-2b16-49e3-d74d-4c83d754b041"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "32"
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "criterion = nn.MSELoss()\n",
        "optimizer = torch.optim.Adam(model.parameters())"
      ],
      "metadata": {
        "id": "5NTDnrJBytjf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from datetime import datetime\n",
        "# Training loop\n",
        "def train(model, criterion, optimizer, train_loader, valid_loader, epochs):\n",
        "  train_losses = np.zeros(epochs)\n",
        "  test_losses = np.zeros(epochs)\n",
        "\n",
        "  for it in range(epochs):\n",
        "    model.train()   # Model in training mode\n",
        "    t0 = datetime.now()\n",
        "    train_loss = 0\n",
        "    n_train = 0\n",
        "    for batch in train_loader:\n",
        "      # move data to GPU\n",
        "      x_c = batch[0].cuda()\n",
        "      y_c = batch[1].cuda()\n",
        "\n",
        "      # zero the parameter gradients\n",
        "      optimizer.zero_grad()\n",
        "\n",
        "      # Forward pass\n",
        "      outputs = model(x_c)\n",
        "      loss = criterion(outputs, y_c)\n",
        "        \n",
        "      # Backward and optimize\n",
        "      loss.backward()   # Compute Gradients (Back prop)\n",
        "      optimizer.step()  # Update weights(GD/Adam)\n",
        "\n",
        "      train_loss += loss.item()*(batch[0].shape[0])\n",
        "      n_train += batch[0].shape[0]\n",
        "\n",
        "    # Get average train loss\n",
        "    train_loss = train_loss / n_train\n",
        "    \n",
        "    # Evalaute model at the end of each epoch\n",
        "    model.eval()\n",
        "    test_loss = 0\n",
        "    n_test = 0\n",
        "    for batch in valid_loader:\n",
        "      # move data to GPU\n",
        "      x_c = batch[0].cuda()\n",
        "      y_c = batch[1].cuda()\n",
        "\n",
        "      # Forward pass\n",
        "      outputs = model(x_c)\n",
        "      loss = criterion(outputs, y_c)\n",
        "      test_loss += loss.item()*(batch[0].shape[0])\n",
        "      n_test += (batch[0].shape[0])\n",
        "    test_loss = test_loss / n_test\n",
        "\n",
        "    # Save losses\n",
        "    train_losses[it] = train_loss\n",
        "    test_losses[it] = test_loss\n",
        "    \n",
        "    dt = datetime.now() - t0\n",
        "    # print(f'Epoch {it+1}/{epochs}, Train Loss: {train_loss:.4f}, Duration: {dt}')\n",
        "\n",
        "    print(f'Epoch {it+1}/{epochs}, Train Loss: {train_loss:.4f}, \\\n",
        "      Test Loss: {test_loss:.4f}, Duration: {dt}')\n",
        "  \n",
        "  # return train_losses, test_losses\n",
        "  return train_losses"
      ],
      "metadata": {
        "id": "Im_PrmgYytjf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now Train"
      ],
      "metadata": {
        "id": "kgKUoNik4NLC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "train(model, criterion, optimizer, train_loader, valid_loader, epochs=40)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "611d2670-2f45-465a-81d6-8c4568e5410e",
        "id": "IjFKkfLX4UK0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/40, Train Loss: 0.1082,       Test Loss: 0.0668, Duration: 0:00:11.225497\n",
            "Epoch 2/40, Train Loss: 0.0479,       Test Loss: 0.0377, Duration: 0:00:11.240074\n",
            "Epoch 3/40, Train Loss: 0.0384,       Test Loss: 0.0338, Duration: 0:00:11.149384\n",
            "Epoch 4/40, Train Loss: 0.0364,       Test Loss: 0.0317, Duration: 0:00:11.160236\n",
            "Epoch 5/40, Train Loss: 0.0350,       Test Loss: 0.0306, Duration: 0:00:10.831368\n",
            "Epoch 6/40, Train Loss: 0.0340,       Test Loss: 0.0318, Duration: 0:00:10.779368\n",
            "Epoch 7/40, Train Loss: 0.0330,       Test Loss: 0.0314, Duration: 0:00:11.995799\n",
            "Epoch 8/40, Train Loss: 0.0330,       Test Loss: 0.0287, Duration: 0:00:11.397091\n",
            "Epoch 9/40, Train Loss: 0.0326,       Test Loss: 0.0298, Duration: 0:00:11.478709\n",
            "Epoch 10/40, Train Loss: 0.0322,       Test Loss: 0.0290, Duration: 0:00:11.353067\n",
            "Epoch 11/40, Train Loss: 0.0317,       Test Loss: 0.0284, Duration: 0:00:11.243980\n",
            "Epoch 12/40, Train Loss: 0.0313,       Test Loss: 0.0289, Duration: 0:00:11.471582\n",
            "Epoch 13/40, Train Loss: 0.0311,       Test Loss: 0.0276, Duration: 0:00:11.606768\n",
            "Epoch 14/40, Train Loss: 0.0310,       Test Loss: 0.0305, Duration: 0:00:11.393944\n",
            "Epoch 15/40, Train Loss: 0.0309,       Test Loss: 0.0276, Duration: 0:00:11.437992\n",
            "Epoch 16/40, Train Loss: 0.0305,       Test Loss: 0.0279, Duration: 0:00:11.419154\n",
            "Epoch 17/40, Train Loss: 0.0305,       Test Loss: 0.0284, Duration: 0:00:11.426684\n",
            "Epoch 18/40, Train Loss: 0.0302,       Test Loss: 0.0287, Duration: 0:00:11.407777\n",
            "Epoch 19/40, Train Loss: 0.0302,       Test Loss: 0.0271, Duration: 0:00:10.947879\n",
            "Epoch 20/40, Train Loss: 0.0302,       Test Loss: 0.0300, Duration: 0:00:11.216961\n",
            "Epoch 21/40, Train Loss: 0.0300,       Test Loss: 0.0294, Duration: 0:00:11.370601\n",
            "Epoch 22/40, Train Loss: 0.0301,       Test Loss: 0.0275, Duration: 0:00:11.298789\n",
            "Epoch 23/40, Train Loss: 0.0298,       Test Loss: 0.0291, Duration: 0:00:11.338743\n",
            "Epoch 24/40, Train Loss: 0.0297,       Test Loss: 0.0277, Duration: 0:00:12.102708\n",
            "Epoch 25/40, Train Loss: 0.0292,       Test Loss: 0.0280, Duration: 0:00:11.341658\n",
            "Epoch 26/40, Train Loss: 0.0295,       Test Loss: 0.0280, Duration: 0:00:11.422062\n",
            "Epoch 27/40, Train Loss: 0.0296,       Test Loss: 0.0274, Duration: 0:00:11.295334\n",
            "Epoch 28/40, Train Loss: 0.0298,       Test Loss: 0.0287, Duration: 0:00:11.258371\n",
            "Epoch 29/40, Train Loss: 0.0295,       Test Loss: 0.0283, Duration: 0:00:11.276673\n",
            "Epoch 30/40, Train Loss: 0.0295,       Test Loss: 0.0280, Duration: 0:00:11.249459\n",
            "Epoch 31/40, Train Loss: 0.0299,       Test Loss: 0.0264, Duration: 0:00:11.141012\n",
            "Epoch 32/40, Train Loss: 0.0298,       Test Loss: 0.0286, Duration: 0:00:10.674273\n",
            "Epoch 33/40, Train Loss: 0.0293,       Test Loss: 0.0286, Duration: 0:00:11.100898\n",
            "Epoch 34/40, Train Loss: 0.0293,       Test Loss: 0.0270, Duration: 0:00:11.367154\n",
            "Epoch 35/40, Train Loss: 0.0294,       Test Loss: 0.0272, Duration: 0:00:11.289341\n",
            "Epoch 36/40, Train Loss: 0.0294,       Test Loss: 0.0276, Duration: 0:00:11.428017\n",
            "Epoch 37/40, Train Loss: 0.0289,       Test Loss: 0.0282, Duration: 0:00:11.330466\n",
            "Epoch 38/40, Train Loss: 0.0292,       Test Loss: 0.0273, Duration: 0:00:11.321989\n",
            "Epoch 39/40, Train Loss: 0.0289,       Test Loss: 0.0273, Duration: 0:00:11.304622\n",
            "Epoch 40/40, Train Loss: 0.0290,       Test Loss: 0.0280, Duration: 0:00:11.385936\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.10824946, 0.0478946 , 0.03839886, 0.03638623, 0.03498856,\n",
              "       0.03404441, 0.0329901 , 0.0329567 , 0.03256513, 0.03215574,\n",
              "       0.03169486, 0.03126598, 0.03113167, 0.03100631, 0.03085495,\n",
              "       0.03053572, 0.03048201, 0.03015668, 0.03017455, 0.03015544,\n",
              "       0.03004804, 0.03014352, 0.0297592 , 0.02970045, 0.02919757,\n",
              "       0.0295211 , 0.02955804, 0.02978716, 0.02951657, 0.02949136,\n",
              "       0.0298884 , 0.02975253, 0.02926124, 0.02934393, 0.02935099,\n",
              "       0.0293865 , 0.02889427, 0.02924933, 0.02887025, 0.02904667])"
            ]
          },
          "metadata": {},
          "execution_count": 69
        }
      ]
    }
  ]
}