Copy_of_week_7_homework.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "162e66b71fe84721b105c129961c192f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1d5f10c8e0ca4e19ae442bf15ab6e966",
              "IPY_MODEL_5a530ab77aaf40db936bce3bfce5e7f5",
              "IPY_MODEL_fd3f48718b6d47a2a5432cc42c03ec92"
            ],
            "layout": "IPY_MODEL_fd08fb9a785f49a7a15a62a563901469"
          }
        },
        "1d5f10c8e0ca4e19ae442bf15ab6e966": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_228bc6f99723459082ace8338ca0e3a2",
            "placeholder": "​",
            "style": "IPY_MODEL_9ddeacda66134a63bbbe611101977608",
            "value": "Downloading builder script: 100%"
          }
        },
        "5a530ab77aaf40db936bce3bfce5e7f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a4799da52f094b83a52ebe191bdd0a76",
            "max": 4418,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c99a149089da498ebdc6069299e281e4",
            "value": 4418
          }
        },
        "fd3f48718b6d47a2a5432cc42c03ec92": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2d6b8654707740c4b0fbb72b6d0730d0",
            "placeholder": "​",
            "style": "IPY_MODEL_c29c5e23aed048a2a4b69bf25828651b",
            "value": " 4.42k/4.42k [00:00&lt;00:00, 294kB/s]"
          }
        },
        "fd08fb9a785f49a7a15a62a563901469": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "228bc6f99723459082ace8338ca0e3a2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9ddeacda66134a63bbbe611101977608": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a4799da52f094b83a52ebe191bdd0a76": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c99a149089da498ebdc6069299e281e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2d6b8654707740c4b0fbb72b6d0730d0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c29c5e23aed048a2a4b69bf25828651b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c4cae86152504995b6ce736397fdb3bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9019dfa6279e4e7cbc15a5a546cb764b",
              "IPY_MODEL_33e1996d97fe40668b1651e1716a663b",
              "IPY_MODEL_b555c89b0a874844952f06357ea081cd"
            ],
            "layout": "IPY_MODEL_49ede180032a400d8e80431d6ab34674"
          }
        },
        "9019dfa6279e4e7cbc15a5a546cb764b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_06361386cf384795bb93ffeffabdc8bc",
            "placeholder": "​",
            "style": "IPY_MODEL_d8e4fa25618b4e6793c3019943321fc1",
            "value": "Downloading readme: 100%"
          }
        },
        "33e1996d97fe40668b1651e1716a663b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5925018d6e95483492cafa11ea99c534",
            "max": 7733,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_88e4c8a04e0149ec94288a88d6423e56",
            "value": 7733
          }
        },
        "b555c89b0a874844952f06357ea081cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_eb4918d55fd945b387954b188f7f5a79",
            "placeholder": "​",
            "style": "IPY_MODEL_92cff2f5414b42c7826c2d031169b543",
            "value": " 7.73k/7.73k [00:00&lt;00:00, 466kB/s]"
          }
        },
        "49ede180032a400d8e80431d6ab34674": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "06361386cf384795bb93ffeffabdc8bc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d8e4fa25618b4e6793c3019943321fc1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5925018d6e95483492cafa11ea99c534": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "88e4c8a04e0149ec94288a88d6423e56": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "eb4918d55fd945b387954b188f7f5a79": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "92cff2f5414b42c7826c2d031169b543": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9746c74731f546c2880733c39f79d486": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_233e98bb4452472d9536ede4ec4bf83b",
              "IPY_MODEL_c5729f6fd31741568324298e52321b20",
              "IPY_MODEL_459f0ccfe72b47489461e066d9da3e98"
            ],
            "layout": "IPY_MODEL_e627ed8aa2564e84a240a8b6a1e37bf3"
          }
        },
        "233e98bb4452472d9536ede4ec4bf83b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3eee77ea18aa448cb00d20a42d7f147a",
            "placeholder": "​",
            "style": "IPY_MODEL_8428cb0d4fc048a883e521c77893e737",
            "value": "Downloading data: 100%"
          }
        },
        "c5729f6fd31741568324298e52321b20": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_67d4c1e441b74a5694ec34891ff2f55c",
            "max": 60558532,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_da3a3c8cdb174806ba2a6b539b82f485",
            "value": 60558532
          }
        },
        "459f0ccfe72b47489461e066d9da3e98": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4d66a00a0af94f78960aceca9ef0dc0f",
            "placeholder": "​",
            "style": "IPY_MODEL_af4f08ea8d56484ca21003105ecbebb0",
            "value": " 60.6M/60.6M [00:02&lt;00:00, 20.3MB/s]"
          }
        },
        "e627ed8aa2564e84a240a8b6a1e37bf3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3eee77ea18aa448cb00d20a42d7f147a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8428cb0d4fc048a883e521c77893e737": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "67d4c1e441b74a5694ec34891ff2f55c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "da3a3c8cdb174806ba2a6b539b82f485": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "4d66a00a0af94f78960aceca9ef0dc0f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "af4f08ea8d56484ca21003105ecbebb0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "51e31fe6f8544e00bda61e38f80e7954": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f84f49a49665473fbddfd81962fd8fcc",
              "IPY_MODEL_5651eddaee2647cfa14d2c506a725050",
              "IPY_MODEL_ab4474bac07340ba81a332a247714f22"
            ],
            "layout": "IPY_MODEL_2afc4ca00a974e08a175b8ab4bb1c413"
          }
        },
        "f84f49a49665473fbddfd81962fd8fcc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d603c5d123f34fa09fde2520fb1f2ff5",
            "placeholder": "​",
            "style": "IPY_MODEL_e72d8f531d6d419fb66e144204ebc852",
            "value": "Generating train split: "
          }
        },
        "5651eddaee2647cfa14d2c506a725050": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_059ff70b85174d63bd9c95d7a94c4297",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1fdcaf034c4b488d8e5d8247deac2808",
            "value": 1
          }
        },
        "ab4474bac07340ba81a332a247714f22": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bf465be502d549539f6c26faccf64857",
            "placeholder": "​",
            "style": "IPY_MODEL_523f6e1ae31343cb867f4ec1425afb16",
            "value": " 614093/0 [01:15&lt;00:00, 5649.64 examples/s]"
          }
        },
        "2afc4ca00a974e08a175b8ab4bb1c413": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d603c5d123f34fa09fde2520fb1f2ff5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e72d8f531d6d419fb66e144204ebc852": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "059ff70b85174d63bd9c95d7a94c4297": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "20px"
          }
        },
        "1fdcaf034c4b488d8e5d8247deac2808": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "bf465be502d549539f6c26faccf64857": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "523f6e1ae31343cb867f4ec1425afb16": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5b241774100a4058a51bc04173243195": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a6ff03ab4e64498aa7dddc9636b29130",
              "IPY_MODEL_58f1f3c24f47494784af10d2273e05aa",
              "IPY_MODEL_f328c8cb9a624fc79fa386c2ac96240b"
            ],
            "layout": "IPY_MODEL_136e8827528844d69a569bdcd256bf53"
          }
        },
        "a6ff03ab4e64498aa7dddc9636b29130": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6ca58f514b604620b3ab47524e6b476c",
            "placeholder": "​",
            "style": "IPY_MODEL_325ae90ae4434c1db2a6ecc16cf7dabb",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "58f1f3c24f47494784af10d2273e05aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5d50ad80abca40838bce1d86e8f13ef8",
            "max": 42,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_993ac5e595bd44eb8df4bdbeb06c401e",
            "value": 42
          }
        },
        "f328c8cb9a624fc79fa386c2ac96240b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_50e80268ae8d4155ab32afc736897b8a",
            "placeholder": "​",
            "style": "IPY_MODEL_f2e5c215a4434094b0ac2f9a98fb4f60",
            "value": " 42.0/42.0 [00:00&lt;00:00, 1.20kB/s]"
          }
        },
        "136e8827528844d69a569bdcd256bf53": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6ca58f514b604620b3ab47524e6b476c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "325ae90ae4434c1db2a6ecc16cf7dabb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5d50ad80abca40838bce1d86e8f13ef8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "993ac5e595bd44eb8df4bdbeb06c401e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "50e80268ae8d4155ab32afc736897b8a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f2e5c215a4434094b0ac2f9a98fb4f60": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5a348f222f594320bed337bd3da326eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8284191c67e64637bbea04a40498af2b",
              "IPY_MODEL_d578a2302ffd4b93878561d634ec277f",
              "IPY_MODEL_32845631556341ecb692b9b1d78b534f"
            ],
            "layout": "IPY_MODEL_ef9897195710498089661711131659df"
          }
        },
        "8284191c67e64637bbea04a40498af2b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_602a14101ca94718b98cf2336dc919f8",
            "placeholder": "​",
            "style": "IPY_MODEL_b11e2d4b824c42dfb81376dc2d773fcf",
            "value": "config.json: 100%"
          }
        },
        "d578a2302ffd4b93878561d634ec277f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_666649803d634937be985a70e79aa209",
            "max": 1416,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d237ad83813c4578890c5b7a3d16a3a9",
            "value": 1416
          }
        },
        "32845631556341ecb692b9b1d78b534f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bd0f470ce26d41679501de6868b65006",
            "placeholder": "​",
            "style": "IPY_MODEL_00e885ae0ee14da7bedbf8e4c9e210b8",
            "value": " 1.42k/1.42k [00:00&lt;00:00, 42.4kB/s]"
          }
        },
        "ef9897195710498089661711131659df": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "602a14101ca94718b98cf2336dc919f8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b11e2d4b824c42dfb81376dc2d773fcf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "666649803d634937be985a70e79aa209": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d237ad83813c4578890c5b7a3d16a3a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "bd0f470ce26d41679501de6868b65006": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "00e885ae0ee14da7bedbf8e4c9e210b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "66af315dd53e4e23a36ba3a7f57d5f0c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_52fb5a70bd5e4dfe9c8819b7447135d3",
              "IPY_MODEL_b94329cec89d40a1950b76d0b157bb1b",
              "IPY_MODEL_d401d2bcb122471f9757b432d8950a81"
            ],
            "layout": "IPY_MODEL_bb52cdaecdc84c439c1e7c1edc0e2ec2"
          }
        },
        "52fb5a70bd5e4dfe9c8819b7447135d3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6e7b7cdad0fb48b79f2a1c7ce11c857b",
            "placeholder": "​",
            "style": "IPY_MODEL_27c2f30428d94fa19296eafa69a8a792",
            "value": "source.spm: 100%"
          }
        },
        "b94329cec89d40a1950b76d0b157bb1b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d0e6dd55cbd74bcda3e73496680f9a7e",
            "max": 778395,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8cd774f7212f48b799e2901e063edec8",
            "value": 778395
          }
        },
        "d401d2bcb122471f9757b432d8950a81": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2a5ab222fd7e4a90ba217e2363d5d214",
            "placeholder": "​",
            "style": "IPY_MODEL_c76b7961dfda44ed8e3e3c9cf99ecec3",
            "value": " 778k/778k [00:00&lt;00:00, 7.70MB/s]"
          }
        },
        "bb52cdaecdc84c439c1e7c1edc0e2ec2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6e7b7cdad0fb48b79f2a1c7ce11c857b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "27c2f30428d94fa19296eafa69a8a792": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d0e6dd55cbd74bcda3e73496680f9a7e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8cd774f7212f48b799e2901e063edec8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2a5ab222fd7e4a90ba217e2363d5d214": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c76b7961dfda44ed8e3e3c9cf99ecec3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "210b4e391fa74c079114df6226657679": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_13a40da950e2483f8726977534867ff4",
              "IPY_MODEL_a20aa6c29670404392e44c54ae76a93c",
              "IPY_MODEL_a58b55e9cd7d47668ce5e1ce39a3e81e"
            ],
            "layout": "IPY_MODEL_a230a4e039c24889a652f551d12f62ce"
          }
        },
        "13a40da950e2483f8726977534867ff4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_58d76acd948f45a594a842d5d03b063c",
            "placeholder": "​",
            "style": "IPY_MODEL_a70fa60a302c41fcb50b74c352495499",
            "value": "target.spm: 100%"
          }
        },
        "a20aa6c29670404392e44c54ae76a93c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_286d0286042e4f89b2803c7205798e49",
            "max": 802397,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_fc92c7eaac7443238400c15872538f1b",
            "value": 802397
          }
        },
        "a58b55e9cd7d47668ce5e1ce39a3e81e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4f45777ef5bc456fb0e49ef3185e09fd",
            "placeholder": "​",
            "style": "IPY_MODEL_faac7bc36639472b8552be550c1114fb",
            "value": " 802k/802k [00:00&lt;00:00, 11.7MB/s]"
          }
        },
        "a230a4e039c24889a652f551d12f62ce": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "58d76acd948f45a594a842d5d03b063c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a70fa60a302c41fcb50b74c352495499": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "286d0286042e4f89b2803c7205798e49": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fc92c7eaac7443238400c15872538f1b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "4f45777ef5bc456fb0e49ef3185e09fd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "faac7bc36639472b8552be550c1114fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "525dfac4c6fc416ab9c0cfbcc5511f51": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_20115b99764b439fb979f39e68b00e40",
              "IPY_MODEL_942aea8c861243448c738b73377beb9e",
              "IPY_MODEL_738e799705b74ed49309c20cc00b8631"
            ],
            "layout": "IPY_MODEL_60093b14230744f1a556d1eb606a8c1a"
          }
        },
        "20115b99764b439fb979f39e68b00e40": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a33e47f7b6614be19b5168776bdc2292",
            "placeholder": "​",
            "style": "IPY_MODEL_2715e70159bc411ea1b19fb5052d8bc7",
            "value": "vocab.json: 100%"
          }
        },
        "942aea8c861243448c738b73377beb9e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_12ded31fff924970be1f70452ced3a1d",
            "max": 1339166,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_194354232bd04bdeaa6e32f595e52d74",
            "value": 1339166
          }
        },
        "738e799705b74ed49309c20cc00b8631": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a7d68a759b2e4710a8d00b6cefdf0d32",
            "placeholder": "​",
            "style": "IPY_MODEL_53abecd093bb4a4b8a9e229a2361803f",
            "value": " 1.34M/1.34M [00:00&lt;00:00, 19.0MB/s]"
          }
        },
        "60093b14230744f1a556d1eb606a8c1a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a33e47f7b6614be19b5168776bdc2292": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2715e70159bc411ea1b19fb5052d8bc7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "12ded31fff924970be1f70452ced3a1d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "194354232bd04bdeaa6e32f595e52d74": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a7d68a759b2e4710a8d00b6cefdf0d32": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "53abecd093bb4a4b8a9e229a2361803f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1fa67df3de4048fdbe187149d0858dcb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0771f03ce4d24ec48fef2891bc80b992",
              "IPY_MODEL_5ab6bf536e9a4e2993546e82fc857cf5",
              "IPY_MODEL_2f4266f8db964a95ac936e384a23dd3f"
            ],
            "layout": "IPY_MODEL_45e69aece3e2493b998455177a935811"
          }
        },
        "0771f03ce4d24ec48fef2891bc80b992": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6ceb05c9c2004267988bc696779a33d9",
            "placeholder": "​",
            "style": "IPY_MODEL_714b90b37c9a4bde9303f3a89fe945d2",
            "value": "Map: 100%"
          }
        },
        "5ab6bf536e9a4e2993546e82fc857cf5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2a770a463aa747c59e6234c9453827a7",
            "max": 491274,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c85cedef0f664e0e86304b611e1633dc",
            "value": 491274
          }
        },
        "2f4266f8db964a95ac936e384a23dd3f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c8782b1f85984448b103e2749cc9c985",
            "placeholder": "​",
            "style": "IPY_MODEL_01f0ffee24004971bb00154c782d66c7",
            "value": " 491274/491274 [02:16&lt;00:00, 2526.07 examples/s]"
          }
        },
        "45e69aece3e2493b998455177a935811": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6ceb05c9c2004267988bc696779a33d9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "714b90b37c9a4bde9303f3a89fe945d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2a770a463aa747c59e6234c9453827a7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c85cedef0f664e0e86304b611e1633dc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c8782b1f85984448b103e2749cc9c985": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "01f0ffee24004971bb00154c782d66c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8c3f8d86192642268477dd770bc3524d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d9c1a1d7eed44b388b0a06353c545d7e",
              "IPY_MODEL_7b6ffb09decc49d59c2fcda3e91e7c82",
              "IPY_MODEL_517f1514a36549e4a8cf26afc93d719e"
            ],
            "layout": "IPY_MODEL_85d4f95434ed45b887b28da61c6e16a2"
          }
        },
        "d9c1a1d7eed44b388b0a06353c545d7e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4d7be2f573ec405a9167ebf6f33eb095",
            "placeholder": "​",
            "style": "IPY_MODEL_df232a66035c479fbcd50efa578d51b4",
            "value": "Map: 100%"
          }
        },
        "7b6ffb09decc49d59c2fcda3e91e7c82": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_398e21a6d10e4ab6ba30794bfc3322b8",
            "max": 122819,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_69c869b9a4cb4d9c8191159309e574c9",
            "value": 122819
          }
        },
        "517f1514a36549e4a8cf26afc93d719e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_81cd76221af6403b8a2311b4b515924b",
            "placeholder": "​",
            "style": "IPY_MODEL_8e6cc270f58d4c3785e199cddfa56c4a",
            "value": " 122819/122819 [00:32&lt;00:00, 4001.76 examples/s]"
          }
        },
        "85d4f95434ed45b887b28da61c6e16a2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4d7be2f573ec405a9167ebf6f33eb095": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "df232a66035c479fbcd50efa578d51b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "398e21a6d10e4ab6ba30794bfc3322b8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "69c869b9a4cb4d9c8191159309e574c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "81cd76221af6403b8a2311b4b515924b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8e6cc270f58d4c3785e199cddfa56c4a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4ed33cb580cc42cdb3848b96cf8ef1cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_74af0bd6f61a47eaba701167c2c5cf08",
              "IPY_MODEL_06af635964cf4726be454b160d2b82a1",
              "IPY_MODEL_4a100a20dac349f480943e050613ec2d"
            ],
            "layout": "IPY_MODEL_2723a29af22448f181cbdea77ea8b265"
          }
        },
        "74af0bd6f61a47eaba701167c2c5cf08": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7782d3824b774122bc12ab5c01f5452a",
            "placeholder": "​",
            "style": "IPY_MODEL_6b5d52fcc1b64295ac9dd7adfd7d9d82",
            "value": "Map: 100%"
          }
        },
        "06af635964cf4726be454b160d2b82a1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4226569a152b40b691f8509c1ba06c50",
            "max": 491274,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_60af036a5ab543629c84c66ba2df3294",
            "value": 491274
          }
        },
        "4a100a20dac349f480943e050613ec2d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6034d95dc77748819d6623376a2472b2",
            "placeholder": "​",
            "style": "IPY_MODEL_d181948a0c0348fdb3ddb89d421a0d5f",
            "value": " 491274/491274 [01:06&lt;00:00, 8552.29 examples/s]"
          }
        },
        "2723a29af22448f181cbdea77ea8b265": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7782d3824b774122bc12ab5c01f5452a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6b5d52fcc1b64295ac9dd7adfd7d9d82": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4226569a152b40b691f8509c1ba06c50": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "60af036a5ab543629c84c66ba2df3294": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6034d95dc77748819d6623376a2472b2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d181948a0c0348fdb3ddb89d421a0d5f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "37a52b1167af4dfeaa2f9cb386757773": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_26ffd43531044dfc9859761a9a3ca382",
              "IPY_MODEL_0dc204ea17e047e78db92d022104ae8c",
              "IPY_MODEL_e08c46ea64bd473bac84303c57ccee3e"
            ],
            "layout": "IPY_MODEL_e34c12434d4e48b685f4c69d3ac6559d"
          }
        },
        "26ffd43531044dfc9859761a9a3ca382": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a41b0c89b4684a59b756f5c38aac4b92",
            "placeholder": "​",
            "style": "IPY_MODEL_137a360d104c4064bc8d2cbd7c9ed9f2",
            "value": "Map: 100%"
          }
        },
        "0dc204ea17e047e78db92d022104ae8c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e41b3531545442e489dc5c6deb61d10a",
            "max": 122819,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_aae133f0be9d47a08a08e280cfc2cb7b",
            "value": 122819
          }
        },
        "e08c46ea64bd473bac84303c57ccee3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_da8f9d00231644d1be3012e22cb0828f",
            "placeholder": "​",
            "style": "IPY_MODEL_953a18312c83414283cb8e74cf0938ef",
            "value": " 122819/122819 [00:14&lt;00:00, 9942.76 examples/s]"
          }
        },
        "e34c12434d4e48b685f4c69d3ac6559d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a41b0c89b4684a59b756f5c38aac4b92": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "137a360d104c4064bc8d2cbd7c9ed9f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e41b3531545442e489dc5c6deb61d10a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "aae133f0be9d47a08a08e280cfc2cb7b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "da8f9d00231644d1be3012e22cb0828f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "953a18312c83414283cb8e74cf0938ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fe1b9fda9a3847a0806158c4689e060f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_150e7c620f6746f7bc0ea2b6c94713cb",
              "IPY_MODEL_52c025f2ab3247ffae9233ec215b373e",
              "IPY_MODEL_c95368b3cdc94a3aa032d55a1b76ff78"
            ],
            "layout": "IPY_MODEL_7e4c1f72cfa8426c95953cdad952966a"
          }
        },
        "150e7c620f6746f7bc0ea2b6c94713cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_18eee73eeb324310a4d14595a84562fa",
            "placeholder": "​",
            "style": "IPY_MODEL_90e86d07551344daa0db665399af0b12",
            "value": "Downloading builder script: 100%"
          }
        },
        "52c025f2ab3247ffae9233ec215b373e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3dc7bd05088a48e98568c8bdfb836946",
            "max": 8146,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_33d55f89ebc648a1aa96adf7632cdcd5",
            "value": 8146
          }
        },
        "c95368b3cdc94a3aa032d55a1b76ff78": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_25d5ab5774284b5095b9adab13889906",
            "placeholder": "​",
            "style": "IPY_MODEL_0b7aed03b8c44aa4938771fcdcd6f98e",
            "value": " 8.15k/8.15k [00:00&lt;00:00, 290kB/s]"
          }
        },
        "7e4c1f72cfa8426c95953cdad952966a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "18eee73eeb324310a4d14595a84562fa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "90e86d07551344daa0db665399af0b12": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3dc7bd05088a48e98568c8bdfb836946": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "33d55f89ebc648a1aa96adf7632cdcd5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "25d5ab5774284b5095b9adab13889906": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0b7aed03b8c44aa4938771fcdcd6f98e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "## Sequence-to-Sequence Model for Machine Translation"
      ],
      "metadata": {
        "id": "ATipbjVZ_HRT"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "In this assignment we will apply transformer-based model for machine translation.\n",
        "\n",
        "We will use tools from `transformers`, `datasets` and `evaluate` libraries by HuggingFace.\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "XkNv-_dp_fOJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install datasets evaluate transformers[sentencepiece]\n",
        "!pip install accelerate"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "t2q3z1el9Hzn",
        "outputId": "ed9c86b5-abe2-49ac-a15e-346c70bce7f4"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting datasets\n",
            "  Downloading datasets-2.18.0-py3-none-any.whl (510 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m510.5/510.5 kB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting evaluate\n",
            "  Downloading evaluate-0.4.1-py3-none-any.whl (84 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.1/84.1 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: transformers[sentencepiece] in /usr/local/lib/python3.10/dist-packages (4.38.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from datasets) (3.13.1)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from datasets) (1.25.2)\n",
            "Requirement already satisfied: pyarrow>=12.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (14.0.2)\n",
            "Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets) (0.6)\n",
            "Collecting dill<0.3.9,>=0.3.0 (from datasets)\n",
            "  Downloading dill-0.3.8-py3-none-any.whl (116 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m7.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (1.5.3)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (4.66.2)\n",
            "Collecting xxhash (from datasets)\n",
            "  Downloading xxhash-3.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m7.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting multiprocess (from datasets)\n",
            "  Downloading multiprocess-0.70.16-py310-none-any.whl (134 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m9.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: fsspec[http]<=2024.2.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2023.6.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.9.3)\n",
            "Requirement already satisfied: huggingface-hub>=0.19.4 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.20.3)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets) (24.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (6.0.1)\n",
            "Collecting responses<0.19 (from evaluate)\n",
            "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece]) (2023.12.25)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece]) (0.15.2)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece]) (0.4.2)\n",
            "Requirement already satisfied: sentencepiece!=0.1.92,>=0.1.91 in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece]) (0.1.99)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece]) (3.20.3)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->datasets) (4.10.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2024.2.2)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2023.4)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n",
            "Installing collected packages: xxhash, dill, responses, multiprocess, datasets, evaluate\n",
            "Successfully installed datasets-2.18.0 dill-0.3.8 evaluate-0.4.1 multiprocess-0.70.16 responses-0.18.0 xxhash-3.4.1\n",
            "Collecting accelerate\n",
            "  Downloading accelerate-0.28.0-py3-none-any.whl (290 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m290.1/290.1 kB\u001b[0m \u001b[31m4.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from accelerate) (1.25.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (24.0)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate) (5.9.5)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from accelerate) (6.0.1)\n",
            "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (2.2.1+cu121)\n",
            "Requirement already satisfied: huggingface-hub in /usr/local/lib/python3.10/dist-packages (from accelerate) (0.20.3)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from accelerate) (0.4.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.13.1)\n",
            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (4.10.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.1.3)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2023.6.0)\n",
            "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch>=1.10.0->accelerate)\n",
            "  Downloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.7/23.7 MB\u001b[0m \u001b[31m30.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cuda-runtime-cu12==12.1.105 (from torch>=1.10.0->accelerate)\n",
            "  Downloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m823.6/823.6 kB\u001b[0m \u001b[31m28.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cuda-cupti-cu12==12.1.105 (from torch>=1.10.0->accelerate)\n",
            "  Downloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.1/14.1 MB\u001b[0m \u001b[31m24.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cudnn-cu12==8.9.2.26 (from torch>=1.10.0->accelerate)\n",
            "  Downloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m731.7/731.7 MB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cublas-cu12==12.1.3.1 (from torch>=1.10.0->accelerate)\n",
            "  Downloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m410.6/410.6 MB\u001b[0m \u001b[31m2.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cufft-cu12==11.0.2.54 (from torch>=1.10.0->accelerate)\n",
            "  Downloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.6/121.6 MB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-curand-cu12==10.3.2.106 (from torch>=1.10.0->accelerate)\n",
            "  Downloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.5/56.5 MB\u001b[0m \u001b[31m11.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cusolver-cu12==11.4.5.107 (from torch>=1.10.0->accelerate)\n",
            "  Downloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.2/124.2 MB\u001b[0m \u001b[31m8.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cusparse-cu12==12.1.0.106 (from torch>=1.10.0->accelerate)\n",
            "  Downloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m196.0/196.0 MB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-nccl-cu12==2.19.3 (from torch>=1.10.0->accelerate)\n",
            "  Downloading nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl (166.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m166.0/166.0 MB\u001b[0m \u001b[31m7.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-nvtx-cu12==12.1.105 (from torch>=1.10.0->accelerate)\n",
            "  Downloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.1/99.1 kB\u001b[0m \u001b[31m15.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2.2.0)\n",
            "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.10.0->accelerate)\n",
            "  Downloading nvidia_nvjitlink_cu12-12.4.99-py3-none-manylinux2014_x86_64.whl (21.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.1/21.1 MB\u001b[0m \u001b[31m77.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->accelerate) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->accelerate) (4.66.2)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.5)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (2024.2.2)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\n",
            "Installing collected packages: nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, nvidia-cusparse-cu12, nvidia-cudnn-cu12, nvidia-cusolver-cu12, accelerate\n",
            "Successfully installed accelerate-0.28.0 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.19.3 nvidia-nvjitlink-cu12-12.4.99 nvidia-nvtx-cu12-12.1.105\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
      ],
      "metadata": {
        "id": "G912QOo99HvL"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Data\n"
      ],
      "metadata": {
        "id": "fi3F7vHeBSfh"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Today there's a number of pretrained machine translation models for different languages. But in practice we could face a task of domain specific translation where general pretrained models sometimes fail to show good performance.\n",
        "\n",
        "Let's try to fine-tune pretrained machine translation model on medical data. We will use PubMed dataset with English-French texts from `datasets` library."
      ],
      "metadata": {
        "id": "OfkUGVUzjCzS"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "5_sVLoKI2qbA",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 344,
          "referenced_widgets": [
            "162e66b71fe84721b105c129961c192f",
            "1d5f10c8e0ca4e19ae442bf15ab6e966",
            "5a530ab77aaf40db936bce3bfce5e7f5",
            "fd3f48718b6d47a2a5432cc42c03ec92",
            "fd08fb9a785f49a7a15a62a563901469",
            "228bc6f99723459082ace8338ca0e3a2",
            "9ddeacda66134a63bbbe611101977608",
            "a4799da52f094b83a52ebe191bdd0a76",
            "c99a149089da498ebdc6069299e281e4",
            "2d6b8654707740c4b0fbb72b6d0730d0",
            "c29c5e23aed048a2a4b69bf25828651b",
            "c4cae86152504995b6ce736397fdb3bc",
            "9019dfa6279e4e7cbc15a5a546cb764b",
            "33e1996d97fe40668b1651e1716a663b",
            "b555c89b0a874844952f06357ea081cd",
            "49ede180032a400d8e80431d6ab34674",
            "06361386cf384795bb93ffeffabdc8bc",
            "d8e4fa25618b4e6793c3019943321fc1",
            "5925018d6e95483492cafa11ea99c534",
            "88e4c8a04e0149ec94288a88d6423e56",
            "eb4918d55fd945b387954b188f7f5a79",
            "92cff2f5414b42c7826c2d031169b543",
            "9746c74731f546c2880733c39f79d486",
            "233e98bb4452472d9536ede4ec4bf83b",
            "c5729f6fd31741568324298e52321b20",
            "459f0ccfe72b47489461e066d9da3e98",
            "e627ed8aa2564e84a240a8b6a1e37bf3",
            "3eee77ea18aa448cb00d20a42d7f147a",
            "8428cb0d4fc048a883e521c77893e737",
            "67d4c1e441b74a5694ec34891ff2f55c",
            "da3a3c8cdb174806ba2a6b539b82f485",
            "4d66a00a0af94f78960aceca9ef0dc0f",
            "af4f08ea8d56484ca21003105ecbebb0",
            "51e31fe6f8544e00bda61e38f80e7954",
            "f84f49a49665473fbddfd81962fd8fcc",
            "5651eddaee2647cfa14d2c506a725050",
            "ab4474bac07340ba81a332a247714f22",
            "2afc4ca00a974e08a175b8ab4bb1c413",
            "d603c5d123f34fa09fde2520fb1f2ff5",
            "e72d8f531d6d419fb66e144204ebc852",
            "059ff70b85174d63bd9c95d7a94c4297",
            "1fdcaf034c4b488d8e5d8247deac2808",
            "bf465be502d549539f6c26faccf64857",
            "523f6e1ae31343cb867f4ec1425afb16"
          ]
        },
        "outputId": "bccefb11-be77-452e-ba44-8ea31f3ff865"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/datasets/load.py:1461: FutureWarning: The repository for qanastek/WMT-16-PubMed contains custom code which must be executed to correctly load the dataset. You can inspect the repository content at https://hf.co/datasets/qanastek/WMT-16-PubMed\n",
            "You can avoid this message in future by passing the argument `trust_remote_code=True`.\n",
            "Passing `trust_remote_code=True` will be mandatory to load this dataset from the next major release of `datasets`.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading builder script:   0%|          | 0.00/4.42k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "162e66b71fe84721b105c129961c192f"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading readme:   0%|          | 0.00/7.73k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c4cae86152504995b6ce736397fdb3bc"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data:   0%|          | 0.00/60.6M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9746c74731f546c2880733c39f79d486"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating train split: 0 examples [00:00, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "51e31fe6f8544e00bda61e38f80e7954"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "from datasets import load_dataset\n",
        "\n",
        "raw_datasets = load_dataset(\"qanastek/WMT-16-PubMed\", lang1=\"en\", lang2=\"fr\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "raw_datasets"
      ],
      "metadata": {
        "id": "Pj_Gu66X5Bt-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ae16efec-85b0-4980-91e7-ddf251a7a2d7"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['translation'],\n",
              "        num_rows: 614093\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We will split the dataset into train and validation part."
      ],
      "metadata": {
        "id": "Cwv8170KlHjO"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "Sv4a9Xyj2qbC",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "644d11cc-966e-4704-d03b-3197e219f33c"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['translation'],\n",
              "        num_rows: 491274\n",
              "    })\n",
              "    test: Dataset({\n",
              "        features: ['translation'],\n",
              "        num_rows: 122819\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ],
      "source": [
        "split_datasets = raw_datasets[\"train\"].train_test_split(train_size=0.8, seed=20)\n",
        "split_datasets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "zeAchgXM2qbD"
      },
      "outputs": [],
      "source": [
        "split_datasets[\"validation\"] = split_datasets.pop(\"test\")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Tokenization"
      ],
      "metadata": {
        "id": "3qCd2eTNly6-"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "We will use transformer model `Helsinki-NLP/opus-mt-en-fr` pretrained for machine translation from English to French.\n",
        "\n",
        "Transformer models use BPE tokenization and tokenizer is trained simultaneously with model training. So we need to load tokenizer from the pretrained model we are going to apply."
      ],
      "metadata": {
        "id": "yR3oHTqSBjN8"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "zjb-iNZP2qbF",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 233,
          "referenced_widgets": [
            "5b241774100a4058a51bc04173243195",
            "a6ff03ab4e64498aa7dddc9636b29130",
            "58f1f3c24f47494784af10d2273e05aa",
            "f328c8cb9a624fc79fa386c2ac96240b",
            "136e8827528844d69a569bdcd256bf53",
            "6ca58f514b604620b3ab47524e6b476c",
            "325ae90ae4434c1db2a6ecc16cf7dabb",
            "5d50ad80abca40838bce1d86e8f13ef8",
            "993ac5e595bd44eb8df4bdbeb06c401e",
            "50e80268ae8d4155ab32afc736897b8a",
            "f2e5c215a4434094b0ac2f9a98fb4f60",
            "5a348f222f594320bed337bd3da326eb",
            "8284191c67e64637bbea04a40498af2b",
            "d578a2302ffd4b93878561d634ec277f",
            "32845631556341ecb692b9b1d78b534f",
            "ef9897195710498089661711131659df",
            "602a14101ca94718b98cf2336dc919f8",
            "b11e2d4b824c42dfb81376dc2d773fcf",
            "666649803d634937be985a70e79aa209",
            "d237ad83813c4578890c5b7a3d16a3a9",
            "bd0f470ce26d41679501de6868b65006",
            "00e885ae0ee14da7bedbf8e4c9e210b8",
            "66af315dd53e4e23a36ba3a7f57d5f0c",
            "52fb5a70bd5e4dfe9c8819b7447135d3",
            "b94329cec89d40a1950b76d0b157bb1b",
            "d401d2bcb122471f9757b432d8950a81",
            "bb52cdaecdc84c439c1e7c1edc0e2ec2",
            "6e7b7cdad0fb48b79f2a1c7ce11c857b",
            "27c2f30428d94fa19296eafa69a8a792",
            "d0e6dd55cbd74bcda3e73496680f9a7e",
            "8cd774f7212f48b799e2901e063edec8",
            "2a5ab222fd7e4a90ba217e2363d5d214",
            "c76b7961dfda44ed8e3e3c9cf99ecec3",
            "210b4e391fa74c079114df6226657679",
            "13a40da950e2483f8726977534867ff4",
            "a20aa6c29670404392e44c54ae76a93c",
            "a58b55e9cd7d47668ce5e1ce39a3e81e",
            "a230a4e039c24889a652f551d12f62ce",
            "58d76acd948f45a594a842d5d03b063c",
            "a70fa60a302c41fcb50b74c352495499",
            "286d0286042e4f89b2803c7205798e49",
            "fc92c7eaac7443238400c15872538f1b",
            "4f45777ef5bc456fb0e49ef3185e09fd",
            "faac7bc36639472b8552be550c1114fb",
            "525dfac4c6fc416ab9c0cfbcc5511f51",
            "20115b99764b439fb979f39e68b00e40",
            "942aea8c861243448c738b73377beb9e",
            "738e799705b74ed49309c20cc00b8631",
            "60093b14230744f1a556d1eb606a8c1a",
            "a33e47f7b6614be19b5168776bdc2292",
            "2715e70159bc411ea1b19fb5052d8bc7",
            "12ded31fff924970be1f70452ced3a1d",
            "194354232bd04bdeaa6e32f595e52d74",
            "a7d68a759b2e4710a8d00b6cefdf0d32",
            "53abecd093bb4a4b8a9e229a2361803f"
          ]
        },
        "outputId": "17b3a626-3912-40a6-b0fe-82eba309624b"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/42.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5b241774100a4058a51bc04173243195"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/1.42k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5a348f222f594320bed337bd3da326eb"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "source.spm:   0%|          | 0.00/778k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "66af315dd53e4e23a36ba3a7f57d5f0c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "target.spm:   0%|          | 0.00/802k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "210b4e391fa74c079114df6226657679"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "vocab.json:   0%|          | 0.00/1.34M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "525dfac4c6fc416ab9c0cfbcc5511f51"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/models/marian/tokenization_marian.py:197: UserWarning: Recommended: pip install sacremoses.\n",
            "  warnings.warn(\"Recommended: pip install sacremoses.\")\n"
          ]
        }
      ],
      "source": [
        "from transformers import AutoTokenizer\n",
        "\n",
        "model_checkpoint = \"Helsinki-NLP/opus-mt-en-fr\"\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, return_tensors=\"pt\")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Check the results of tokenization for some examples in English and in French."
      ],
      "metadata": {
        "id": "9kCaXtubk4_B"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "text = 'Hyponatremia in the elderly: its role in the frailty'\n",
        "tokenized = tokenizer(text)\n",
        "tokenized"
      ],
      "metadata": {
        "id": "6pAGM6FpkUfV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "70f7d6d0-a724-4cb5-df64-8a5f8a10bff4"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input_ids': [36619, 10484, 158, 27621, 18, 4, 13931, 37, 96, 908, 18, 4, 1231, 17465, 1744, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "To get tokens from the ids returned by tokenizer, we can use `convert_ids_to_tokens`."
      ],
      "metadata": {
        "id": "DWI9D8EnlV5U"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(tokenizer.convert_ids_to_tokens(tokenized['input_ids']))"
      ],
      "metadata": {
        "id": "RO62OKGbmFJg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e0a06332-73f6-48a7-ce62-8e2eba99e336"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['▁Hypo', 'nat', 're', 'mia', '▁in', '▁the', '▁elderly', ':', '▁its', '▁role', '▁in', '▁the', '▁f', 'rail', 'ty', '</s>']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "text = \"L'hyponatrémie du sujet âgé: implication dans la fragilité\"\n",
        "tokenized = tokenizer(text)\n",
        "tokenized"
      ],
      "metadata": {
        "id": "IUoV5fdNnGt2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2d6aed9d-3073-4aa6-a246-770fb559af7b"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input_ids': [87, 6, 8073, 6286, 970, 1605, 10650, 22, 6812, 17117, 18878, 8056, 37, 21549, 31, 8, 30128, 309, 107, 2418, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(tokenizer.convert_ids_to_tokens(tokenized['input_ids']))"
      ],
      "metadata": {
        "id": "fsHycLgnnfNE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d8a776d9-8bff-491c-b070-6e12397618e6"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['▁L', \"'\", 'hy', 'pon', 'at', 'ré', 'mie', '▁du', '▁su', 'jet', '▁â', 'gé', ':', '▁implication', '▁dans', '▁la', '▁fra', 'g', 'il', 'ité', '</s>']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Note that French text was split into very short tokens as the tokenizer didn't recognize the target language. To tokenize target text correctly we need to pass it with the argument `text_target`."
      ],
      "metadata": {
        "id": "DFNPhLB0lvWo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "text = \"L'hyponatrémie du sujet âgé: implication dans la fragilité\"\n",
        "tokenized = tokenizer(text_target=text)"
      ],
      "metadata": {
        "id": "G52DKZwonmyz"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(tokenizer.convert_ids_to_tokens(tokenized['input_ids']))"
      ],
      "metadata": {
        "id": "kH329b2nn0dK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8e046cb4-cc76-4a8f-aa2c-8703407b34c4"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['▁L', \"'\", 'hypo', 'nat', 'r', 'émie', '▁du', '▁sujet', '▁âgé', ':', '▁implication', '▁dans', '▁la', '▁fragilité', '</s>']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Task 1.**\n",
        "\n",
        "*1 point*\n",
        "\n",
        "Write function to preprocess dataset items using tokenizer, don't forget to use `text_target` for target texts.\n",
        "\n",
        "We want to set maximum length for the texts and cut all the texts that appear to be longer. So apply a tokenizer with fixed maximum sequence length using `max_length` parameter and setting `truncation=True`. Taking arbitrary `max_length` is probably not the best idea. Look at the distribution of text lengths and choose it wisely."
      ],
      "metadata": {
        "id": "ZLMM8F9Fmb3b"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "split_datasets"
      ],
      "metadata": {
        "id": "tOMfDOgWnwA5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "55ab21ef-8bd4-443c-e384-7f7be7f7a971"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['translation'],\n",
              "        num_rows: 491274\n",
              "    })\n",
              "    validation: Dataset({\n",
              "        features: ['translation'],\n",
              "        num_rows: 122819\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "split_datasets['train'][0]"
      ],
      "metadata": {
        "id": "FEdUxj9GnwtK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f9ce8496-406f-411f-9b9e-f54a3ddc1dc8"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'translation': {'en': 'Isolated glomus tumor of the left lower limb',\n",
              "  'fr': 'Tumeur glomique isolée du membre inférieur gauche'}}"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "T0-Al4Ak2qbG"
      },
      "outputs": [],
      "source": [
        "# examples -- dataset items\n",
        "def preprocess_function(examples):\n",
        "    examples = examples['translation']\n",
        "    model_inputs = {'input_ids': [], 'attention_mask': [], 'labels': []}\n",
        "    for example in examples:\n",
        "        tokenized_example = tokenizer(example['en'], text_target=example['fr'], max_length=32, truncation=True)\n",
        "        model_inputs['input_ids'].append(tokenized_example['input_ids'])\n",
        "        model_inputs['attention_mask'].append(tokenized_example['attention_mask'])\n",
        "        model_inputs['labels'].append(tokenized_example['labels'])\n",
        "        # model_inputs.append(tokenizer(example['en'], text_target=example['fr'], max_length=32, truncation=True))\n",
        "    # print(model_inputs)\n",
        "    return model_inputs"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Create tokenized dataset by appling preprocessing function to `split_datasets` with `map` method. Set parameter `batched=True` to make computing faster."
      ],
      "metadata": {
        "id": "WffIZ4XVo0_3"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "54ZGYS8x2qbG",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81,
          "referenced_widgets": [
            "1fa67df3de4048fdbe187149d0858dcb",
            "0771f03ce4d24ec48fef2891bc80b992",
            "5ab6bf536e9a4e2993546e82fc857cf5",
            "2f4266f8db964a95ac936e384a23dd3f",
            "45e69aece3e2493b998455177a935811",
            "6ceb05c9c2004267988bc696779a33d9",
            "714b90b37c9a4bde9303f3a89fe945d2",
            "2a770a463aa747c59e6234c9453827a7",
            "c85cedef0f664e0e86304b611e1633dc",
            "c8782b1f85984448b103e2749cc9c985",
            "01f0ffee24004971bb00154c782d66c7",
            "8c3f8d86192642268477dd770bc3524d",
            "d9c1a1d7eed44b388b0a06353c545d7e",
            "7b6ffb09decc49d59c2fcda3e91e7c82",
            "517f1514a36549e4a8cf26afc93d719e",
            "85d4f95434ed45b887b28da61c6e16a2",
            "4d7be2f573ec405a9167ebf6f33eb095",
            "df232a66035c479fbcd50efa578d51b4",
            "398e21a6d10e4ab6ba30794bfc3322b8",
            "69c869b9a4cb4d9c8191159309e574c9",
            "81cd76221af6403b8a2311b4b515924b",
            "8e6cc270f58d4c3785e199cddfa56c4a"
          ]
        },
        "outputId": "f035e79a-8938-4cba-fb75-21412dd3a7c3"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/491274 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "1fa67df3de4048fdbe187149d0858dcb"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/122819 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "8c3f8d86192642268477dd770bc3524d"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "dataset = split_datasets.map(preprocess_function, batched=True)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dataset"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LBRX19UuqkFV",
        "outputId": "7c92835e-aab7-478a-bce3-ae4d20ac26b7"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['translation', 'input_ids', 'attention_mask', 'labels'],\n",
              "        num_rows: 491274\n",
              "    })\n",
              "    validation: Dataset({\n",
              "        features: ['translation', 'input_ids', 'attention_mask', 'labels'],\n",
              "        num_rows: 122819\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dataset['train'][0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Nhv68iERq9nk",
        "outputId": "f20b739e-a8e5-4044-8d41-9d69dbfad5d8"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'translation': {'en': 'Isolated glomus tumor of the left lower limb',\n",
              "  'fr': 'Tumeur glomique isolée du membre inférieur gauche'},\n",
              " 'input_ids': [47,\n",
              "  3790,\n",
              "  2468,\n",
              "  49,\n",
              "  19073,\n",
              "  6732,\n",
              "  40086,\n",
              "  7,\n",
              "  4,\n",
              "  1684,\n",
              "  2803,\n",
              "  33150,\n",
              "  0],\n",
              " 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n",
              " 'labels': [491, 19219, 954, 376, 24087, 33692, 22, 1084, 6792, 4789, 0]}"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def clear_dataset(example):\n",
        "    return {\n",
        "        'input_ids': example['input_ids'],\n",
        "        'attention_mask': example['attention_mask'],\n",
        "        'labels': example['labels']\n",
        "    }\n",
        "\n",
        "dataset['train'] = dataset['train'].map(clear_dataset, remove_columns=['translation'])\n",
        "dataset['validation'] = dataset['validation'].map(clear_dataset, remove_columns=['translation'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81,
          "referenced_widgets": [
            "4ed33cb580cc42cdb3848b96cf8ef1cc",
            "74af0bd6f61a47eaba701167c2c5cf08",
            "06af635964cf4726be454b160d2b82a1",
            "4a100a20dac349f480943e050613ec2d",
            "2723a29af22448f181cbdea77ea8b265",
            "7782d3824b774122bc12ab5c01f5452a",
            "6b5d52fcc1b64295ac9dd7adfd7d9d82",
            "4226569a152b40b691f8509c1ba06c50",
            "60af036a5ab543629c84c66ba2df3294",
            "6034d95dc77748819d6623376a2472b2",
            "d181948a0c0348fdb3ddb89d421a0d5f",
            "37a52b1167af4dfeaa2f9cb386757773",
            "26ffd43531044dfc9859761a9a3ca382",
            "0dc204ea17e047e78db92d022104ae8c",
            "e08c46ea64bd473bac84303c57ccee3e",
            "e34c12434d4e48b685f4c69d3ac6559d",
            "a41b0c89b4684a59b756f5c38aac4b92",
            "137a360d104c4064bc8d2cbd7c9ed9f2",
            "e41b3531545442e489dc5c6deb61d10a",
            "aae133f0be9d47a08a08e280cfc2cb7b",
            "da8f9d00231644d1be3012e22cb0828f",
            "953a18312c83414283cb8e74cf0938ef"
          ]
        },
        "id": "nKvRV-vsieJU",
        "outputId": "740aec6f-d993-4b48-dec9-04b5b61f322f"
      },
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/491274 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "4ed33cb580cc42cdb3848b96cf8ef1cc"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/122819 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "37a52b1167af4dfeaa2f9cb386757773"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dataset"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AQLCEMjYjRV2",
        "outputId": "1f3455cb-3a56-4875-9a89-6ff07917febf"
      },
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['input_ids', 'attention_mask', 'labels'],\n",
              "        num_rows: 491274\n",
              "    })\n",
              "    validation: Dataset({\n",
              "        features: ['input_ids', 'attention_mask', 'labels'],\n",
              "        num_rows: 122819\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dataset[\"train\"][0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ia1VTAGNjT5U",
        "outputId": "4d04dbee-caa0-42df-d74b-c1e3c442769c"
      },
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input_ids': [47,\n",
              "  3790,\n",
              "  2468,\n",
              "  49,\n",
              "  19073,\n",
              "  6732,\n",
              "  40086,\n",
              "  7,\n",
              "  4,\n",
              "  1684,\n",
              "  2803,\n",
              "  33150,\n",
              "  0],\n",
              " 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n",
              " 'labels': [491, 19219, 954, 376, 24087, 33692, 22, 1084, 6792, 4789, 0]}"
            ]
          },
          "metadata": {},
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Model"
      ],
      "metadata": {
        "id": "WhLs45xKjXE1"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can use the model for inference with the help of pipeline from `transformers` library. To initialize pipeline we need to pass task type (in our case `translation`) and model name."
      ],
      "metadata": {
        "id": "pwpTFRvVqICY"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 54,
      "metadata": {
        "id": "t3QcKCkG2qbD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3de51aa3-41e4-4a1a-ab40-a58fe59c788d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/models/marian/tokenization_marian.py:197: UserWarning: Recommended: pip install sacremoses.\n",
            "  warnings.warn(\"Recommended: pip install sacremoses.\")\n"
          ]
        }
      ],
      "source": [
        "from transformers import pipeline\n",
        "\n",
        "model_checkpoint = \"Helsinki-NLP/opus-mt-en-fr\"\n",
        "translator = pipeline(\"translation\", model=model_checkpoint)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Take a look at some examples of translation by pretrained model."
      ],
      "metadata": {
        "id": "eney7S--rhvv"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "translator(\"Clinical efficacy of eculizumab as treatment of gemcitabine-induced thrombotic microangiopathy: A case report\")\n",
        "# Efficacité de l'éculizumab comme traitement de la microangiopathie thrombotique induite par la gemcitabine : à propos d'un cas"
      ],
      "metadata": {
        "id": "CQlhmWZSqGr2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "98b3704a-9e76-4adc-d6f9-3850271439d7"
      },
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'translation_text': \"Efficacité clinique de l'eculizumab comme traitement de la microangiopathie thrombotique induite par la gemcitabine: rapport de cas\"}]"
            ]
          },
          "metadata": {},
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "translator(\"Proliferative verrucous leukoplakia: Three cases and literature review\")\n",
        "# Leucoplasie verruqueuse proliférative : trois observations et revue de la littérature"
      ],
      "metadata": {
        "id": "AmZuV4777ZYQ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2b6f4e38-7dfd-4069-eaf5-6dfa2726b0f1"
      },
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'translation_text': 'Verrucous leukoplakia proliférative : trois cas et revue de la littérature'}]"
            ]
          },
          "metadata": {},
          "execution_count": 56
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "To fine-tune the model on our data we need to load it from pretrained checkpoint.\n",
        "\n",
        "`transformers` library offers wide range of the models for various tasks (for example, `AutoModelForSequenceClassification`, `AutoModelForTokenClassification`, `AutoModelWithLMHead`). For the task of machine translation we will use `AutoModelForSeq2SeqLM` model."
      ],
      "metadata": {
        "id": "NzhPCWe1roS1"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 57,
      "metadata": {
        "id": "y9yylwFW2qbH"
      },
      "outputs": [],
      "source": [
        "from transformers import AutoModelForSeq2SeqLM\n",
        "model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Take a look at the model's structure."
      ],
      "metadata": {
        "id": "LpBW4ICms-qh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model"
      ],
      "metadata": {
        "id": "tqlxHsnxwJtH",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d02d2e3c-da60-4162-bbb3-94408a671eed"
      },
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "MarianMTModel(\n",
              "  (model): MarianModel(\n",
              "    (shared): Embedding(59514, 512, padding_idx=59513)\n",
              "    (encoder): MarianEncoder(\n",
              "      (embed_tokens): Embedding(59514, 512, padding_idx=59513)\n",
              "      (embed_positions): MarianSinusoidalPositionalEmbedding(512, 512)\n",
              "      (layers): ModuleList(\n",
              "        (0-5): 6 x MarianEncoderLayer(\n",
              "          (self_attn): MarianAttention(\n",
              "            (k_proj): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (v_proj): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (q_proj): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
              "          )\n",
              "          (self_attn_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
              "          (activation_fn): SiLU()\n",
              "          (fc1): Linear(in_features=512, out_features=2048, bias=True)\n",
              "          (fc2): Linear(in_features=2048, out_features=512, bias=True)\n",
              "          (final_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (decoder): MarianDecoder(\n",
              "      (embed_tokens): Embedding(59514, 512, padding_idx=59513)\n",
              "      (embed_positions): MarianSinusoidalPositionalEmbedding(512, 512)\n",
              "      (layers): ModuleList(\n",
              "        (0-5): 6 x MarianDecoderLayer(\n",
              "          (self_attn): MarianAttention(\n",
              "            (k_proj): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (v_proj): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (q_proj): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
              "          )\n",
              "          (activation_fn): SiLU()\n",
              "          (self_attn_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
              "          (encoder_attn): MarianAttention(\n",
              "            (k_proj): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (v_proj): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (q_proj): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
              "          )\n",
              "          (encoder_attn_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
              "          (fc1): Linear(in_features=512, out_features=2048, bias=True)\n",
              "          (fc2): Linear(in_features=2048, out_features=512, bias=True)\n",
              "          (final_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (lm_head): Linear(in_features=512, out_features=59514, bias=False)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 58
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Self-Attention\n",
        "\n",
        "Let's get a closer look at the internals of transformer models.\n",
        "\n",
        "Transformer models are based on self-attention mechanism.\n",
        "An attention function aims to assess the relation between the elements of the sequence. The output for each element is computed as a weighted sum of the value vectors of other elements, where the weight for each value is computed by a score function for the query vector (for current element) with the corresponding key (for key-value pairs of other elements). Query, key and value vectors are computed multiplying embeddings by weight matrices $W_q$, $W_k$, $W_v$ that are trained during the model training.\n",
        "\n",
        "$$                                                                         \n",
        "   \\mathrm{Attention}(Q, K, V) = \\mathrm{softmax}(\\frac{QK^T}{\\sqrt{d_k}})V               \n",
        "$$\n",
        "\n",
        "\n",
        "**Task 2 (3 points)**. Implement self-attention mechanism.\n",
        "\n",
        "1. Create random embedding tensor $X$ of dimension $[sequence\\_length, emb\\_dim]$, where $emb\\_dim$ is the dimension of embedding layer in the pretrained model.\n",
        "2. In `SelfAttention` class implement method `initialize_weights` to create random tensors of weights for queries, keys and values $w_q$, $w_k$, $w_v$ of dimension $[emb\\_dim, dim_k]$ (dimension of query, key and value vector will be equal, let's set $dim_k = 64$).\n",
        "3. In `SelfAttention` class implement method `forward` to calculate dot-product attention. The method should take embedding tensor $X$ as an input.\n",
        " - Compute $query$, $key$ and $value$ tensors multiplying embeddings by weights $w_q$, $w_k$, $w_v$ (the result tensors should have dimension $[sequence\\_length, dim_k]$).\n",
        " - Take the dot product of the queries with keys.\n",
        " - Divide the obtaned result by the dimension of key vector.\n",
        " - Use softmax to get the attention weights.\n",
        " - Apply calculated attention weights to the values.\n",
        "\n",
        "It's better to use just tensors, but if you're tempted to use `nn.Linear` instead, do `bias=False`."
      ],
      "metadata": {
        "id": "CXCjvyIUK9BQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "seq_len, emb_dim, dim_k = 10, 10, 64"
      ],
      "metadata": {
        "id": "QoEcxLin6L7W"
      },
      "execution_count": 59,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import torch.nn as nn\n",
        "import torch\n",
        "\n",
        "class SelfAttention(nn.Module):\n",
        "  def __init__(self, seq_len, emb_dim, dim_k):\n",
        "    super(SelfAttention, self).__init__()\n",
        "    self.emb_dim = emb_dim\n",
        "    self.seq_len = seq_len\n",
        "    self.dim_k = dim_k\n",
        "    self.initialize_weights()\n",
        "\n",
        "  def initialize_weights(self):\n",
        "    self.w_q = torch.rand(self.emb_dim, self.dim_k)\n",
        "    self.w_k = torch.rand(self.emb_dim, self.dim_k)\n",
        "    self.w_v = torch.rand(self.emb_dim, self.dim_k)\n",
        "\n",
        "  def forward(self, X):\n",
        "    query = torch.mm(X, self.w_q)\n",
        "    key = torch.mm(X, self.w_k)\n",
        "    value = torch.mm(X, self.w_v)\n",
        "\n",
        "    attn_weights = nn.functional.softmax(torch.mm(query, key.t()) / self.dim_k, dim=1)\n",
        "    output = torch.mm(attn_weights, value)\n",
        "    return output\n"
      ],
      "metadata": {
        "id": "deSt8wXpFRJl"
      },
      "execution_count": 60,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# create embedding tensor X\n",
        "X = torch.rand(seq_len, emb_dim)"
      ],
      "metadata": {
        "id": "pE_GxCyTFXdO"
      },
      "execution_count": 61,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# initialize SelfAttention layer\n",
        "self_attention = SelfAttention(seq_len, emb_dim, dim_k)"
      ],
      "metadata": {
        "id": "k4LMFa3RB05H"
      },
      "execution_count": 62,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# calculate output for X\n",
        "output = self_attention.forward(X)"
      ],
      "metadata": {
        "id": "-E7_osdzFXdQ"
      },
      "execution_count": 63,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Attention Weights"
      ],
      "metadata": {
        "id": "Hh9iACoB1yCx"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now let's check out attention weights of our pretrained model.\n",
        "\n",
        "To access the attention weights we can run `generate` method using parameters `return_dict_in_generate` and `output_attentions`. Let's set `num_beams=1` and `do_sample=False` to apply simple greedy search decoding."
      ],
      "metadata": {
        "id": "wGFxSTRtrD-N"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "input_text = \"Proliferative verrucous leukoplakia: Three cases and literature review\"\n",
        "input_ids = tokenizer.encode(input_text, return_tensors='pt')  # tokenize input text\n",
        "outputs = model.generate(input_ids, return_dict_in_generate=True, output_attentions=True, num_beams=1, do_sample=False)\n"
      ],
      "metadata": {
        "id": "VtXeV-9uc8O3"
      },
      "execution_count": 64,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "`generate` methods returns dict with predicted tokens, and attention values including encoder, decoder and encoder-decoder attention weights."
      ],
      "metadata": {
        "id": "7cAOgzM6uN2y"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "outputs.keys()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vWdtKkpE6k_5",
        "outputId": "6c5986be-160d-4769-889e-d9ecab77ed9d"
      },
      "execution_count": 65,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "odict_keys(['sequences', 'encoder_attentions', 'decoder_attentions', 'cross_attentions', 'past_key_values'])"
            ]
          },
          "metadata": {},
          "execution_count": 65
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "outputs.keys()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MdlQd9Iryb78",
        "outputId": "4c3bc56e-60f2-4a08-dafe-f73044f20032"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "odict_keys(['sequences', 'encoder_attentions', 'decoder_attentions', 'cross_attentions'])"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's look into cross attentions to get the idea how input tokens affect each of the predicted tokens.\n",
        "\n",
        "Each element of `cross_attentions` object contains attention weights for a single generated token. Attention weights for each generated token have tensors of weights for each decoder layer. Attention weights tensor for a single layer has a shape [batch size, number of attention heads, generated sequence length, input sequence length]. As the tensor corresponds to a single generated token, generated sequence length equals 1."
      ],
      "metadata": {
        "id": "bwCxYEQ8unf1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# one element for each generated token\n",
        "len(outputs['cross_attentions'])"
      ],
      "metadata": {
        "id": "KoEqhpbUyedp",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "16c1d01c-5558-4148-f8a2-4a807cfb60f2"
      },
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "24"
            ]
          },
          "metadata": {},
          "execution_count": 66
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# one element for each layer of the decoder\n",
        "len(outputs['cross_attentions'][0])"
      ],
      "metadata": {
        "id": "Yk-BzOd8ylH2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3b2c9d63-e3ba-47f1-f9b1-edfcc9f9e75f"
      },
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "6"
            ]
          },
          "metadata": {},
          "execution_count": 67
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# batch_size, num_heads, generated_length, sequence_length\n",
        "outputs['cross_attentions'][0][0].size()"
      ],
      "metadata": {
        "id": "izdu-BGOym9X",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d9170cfb-a520-4b4f-cc15-c24224d4eeb3"
      },
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([1, 8, 1, 21])"
            ]
          },
          "metadata": {},
          "execution_count": 68
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Task 3.**\n",
        "\n",
        "*3 points*\n",
        "\n",
        "Write code to build a tensor of cross-attention weights with size `[generated_length, input_sequence]` for a fixed attention head and fixed decoder layer (let's take last layer)."
      ],
      "metadata": {
        "id": "mhx4UPggq8ME"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# input_ids -- tensor of input tokens ids\n",
        "# outputs -- output of generate method\n",
        "# attn_head_id -- number of attention head\n",
        "# decoder_layer_id -- number of decoder layer\n",
        "\n",
        "import torch\n",
        "\n",
        "def build_attn(input_ids, outputs, attn_head_id, decoder_layer_id):\n",
        "  result_weights = list()\n",
        "  for cross_attn_weights in outputs.cross_attentions:\n",
        "      required_attention = cross_attn_weights[decoder_layer_id][:, attn_head_id, :, :].squeeze(1)\n",
        "      result_weights.append(required_attention)\n",
        "  attn_tensor = torch.cat(result_weights, dim=0)\n",
        "  return attn_tensor"
      ],
      "metadata": {
        "id": "7kMbii9U0Kr4"
      },
      "execution_count": 69,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Visualize this tensor of cross-attention weights using heatmap to see the relation between input and output tokens.\n",
        "\n",
        "Try the visualization for different input texts and analyze the results."
      ],
      "metadata": {
        "id": "KBgYCxxpvpkQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# input_text = \"Proliferative verrucous leukoplakia: Three cases and literature review\"\n",
        "# input_ids = tokenizer.encode(input_text, return_tensors='pt')  # tokenize input text\n",
        "# print(input_ids)\n",
        "# outputs = model.generate(input_ids, return_dict_in_generate=True, output_attentions=True, num_beams=1, do_sample=False)\n",
        "attn_head_id = 0\n",
        "decoder_layer_id = 1\n",
        "\n",
        "attentions = build_attn(\n",
        "    input_ids, outputs, attn_head_id, decoder_layer_id)\n",
        "print(attentions)\n",
        "plt.matshow(attentions.numpy())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "0-rEf5VD-SrN",
        "outputId": "19f387f7-c242-48a2-8843-9b0d7bac1e89"
      },
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[3.4089e-05, 5.7138e-07, 1.5856e-08, 2.1579e-08, 2.7165e-05, 3.6440e-06,\n",
            "         7.6839e-11, 2.5078e-09, 2.6524e-06, 1.0338e-05, 6.4866e-07, 1.7857e-06,\n",
            "         1.1130e-08, 1.5322e-08, 1.3228e-05, 9.3762e-06, 1.2012e-07, 2.2793e-05,\n",
            "         2.8199e-07, 3.4161e-07, 9.9987e-01],\n",
            "        [2.0619e-03, 1.4205e-03, 1.0409e-03, 4.6442e-04, 1.7581e-03, 1.6812e-03,\n",
            "         6.1490e-05, 7.6695e-05, 9.5924e-04, 1.1683e-03, 1.9683e-04, 1.7571e-04,\n",
            "         3.5008e-04, 2.7127e-04, 9.4939e-04, 1.3106e-03, 1.3255e-03, 2.9774e-04,\n",
            "         1.0532e-03, 4.3007e-04, 9.8295e-01],\n",
            "        [1.3348e-03, 2.5148e-03, 3.5171e-03, 3.5524e-04, 9.0321e-03, 1.0982e-02,\n",
            "         2.3171e-03, 5.6803e-04, 3.7828e-03, 2.3719e-03, 2.7641e-03, 7.8774e-04,\n",
            "         4.0838e-04, 7.2391e-04, 9.9552e-04, 5.9089e-05, 5.5063e-04, 2.2379e-05,\n",
            "         3.1198e-03, 2.2434e-04, 9.5357e-01],\n",
            "        [4.1524e-04, 1.6976e-03, 1.3720e-03, 2.0980e-04, 2.0837e-02, 3.5187e-03,\n",
            "         1.1317e-03, 5.7614e-04, 1.4173e-03, 1.1073e-03, 2.1145e-03, 1.3070e-03,\n",
            "         5.7413e-04, 1.2422e-03, 1.6431e-03, 5.0145e-05, 2.9684e-04, 3.8065e-05,\n",
            "         8.7715e-04, 3.4716e-04, 9.5923e-01],\n",
            "        [3.4998e-03, 1.4733e-03, 3.6615e-03, 3.7385e-03, 1.0407e-02, 2.9969e-03,\n",
            "         6.8499e-03, 1.7747e-03, 2.0019e-03, 4.2714e-04, 1.4444e-03, 4.2447e-04,\n",
            "         5.8301e-04, 1.0106e-03, 1.6492e-03, 8.5473e-05, 6.5830e-04, 1.3709e-04,\n",
            "         1.2633e-03, 1.7811e-04, 9.5574e-01],\n",
            "        [2.1774e-02, 7.3178e-03, 2.2026e-02, 3.9041e-03, 6.8250e-02, 1.1848e-02,\n",
            "         1.9271e-02, 4.3643e-02, 2.8478e-01, 6.6967e-03, 8.2649e-03, 2.1710e-02,\n",
            "         3.2646e-03, 3.5543e-02, 2.1776e-02, 2.0583e-03, 5.7118e-03, 1.9248e-03,\n",
            "         5.6945e-03, 3.2414e-03, 4.0130e-01],\n",
            "        [2.9496e-03, 1.7340e-03, 1.5887e-03, 6.5602e-04, 8.0075e-03, 8.9896e-03,\n",
            "         3.1010e-03, 1.5241e-03, 6.7385e-03, 1.3468e-02, 5.1318e-03, 5.8303e-03,\n",
            "         1.3057e-03, 3.1303e-03, 1.0162e-02, 5.7005e-04, 1.5010e-03, 1.6656e-03,\n",
            "         4.0546e-03, 6.7946e-04, 9.1721e-01],\n",
            "        [5.0599e-03, 4.0816e-03, 5.1304e-03, 2.1848e-03, 4.5386e-03, 2.3635e-03,\n",
            "         2.4136e-03, 2.3011e-03, 3.1036e-03, 1.6038e-03, 1.4244e-02, 7.2224e-03,\n",
            "         1.7707e-03, 1.2373e-03, 1.2313e-03, 6.8840e-05, 1.1227e-04, 6.9082e-05,\n",
            "         8.7465e-04, 5.3842e-05, 9.4033e-01],\n",
            "        [9.2933e-03, 8.3298e-03, 1.5834e-02, 5.1222e-03, 2.4589e-02, 1.4015e-02,\n",
            "         1.1456e-02, 7.3150e-03, 3.1834e-02, 7.8092e-03, 2.7454e-02, 1.6453e-02,\n",
            "         8.0718e-03, 5.3630e-03, 2.2027e-03, 6.1569e-05, 3.4394e-04, 4.0602e-05,\n",
            "         1.3005e-03, 2.7773e-04, 8.0283e-01],\n",
            "        [1.8091e-03, 1.0698e-02, 3.5669e-02, 2.7643e-03, 6.0894e-03, 3.3585e-02,\n",
            "         2.7589e-02, 5.1922e-03, 1.1490e-02, 2.3625e-02, 4.8519e-02, 2.4187e-02,\n",
            "         1.2607e-02, 1.9161e-02, 1.1650e-02, 1.3097e-03, 4.3787e-03, 3.1421e-04,\n",
            "         2.2915e-03, 1.3149e-03, 7.1576e-01],\n",
            "        [4.8266e-03, 6.0332e-03, 2.0992e-02, 5.7600e-03, 1.6150e-02, 8.5264e-03,\n",
            "         1.3898e-02, 2.3880e-03, 5.9501e-03, 4.5136e-03, 9.2590e-03, 5.9476e-03,\n",
            "         6.2331e-03, 7.7967e-02, 1.6627e-02, 5.0212e-03, 1.1047e-03, 1.0475e-03,\n",
            "         1.6204e-03, 8.5884e-04, 7.8528e-01],\n",
            "        [8.7735e-03, 3.7567e-04, 2.8304e-03, 3.5746e-02, 3.9789e-03, 1.3179e-03,\n",
            "         2.7855e-03, 5.3149e-03, 1.0423e-03, 5.6618e-04, 3.3257e-03, 9.8470e-04,\n",
            "         2.4912e-03, 2.6006e-03, 1.4889e-03, 4.3175e-05, 8.4950e-04, 2.3520e-04,\n",
            "         6.2319e-03, 2.8061e-03, 9.1621e-01],\n",
            "        [1.0170e-02, 3.9773e-03, 2.5138e-03, 2.8000e-03, 5.0040e-03, 4.3287e-03,\n",
            "         3.5646e-04, 5.2267e-04, 6.4803e-03, 1.8800e-03, 2.6706e-03, 1.4166e-03,\n",
            "         7.3318e-04, 3.8173e-04, 2.0477e-03, 7.3274e-04, 7.2448e-04, 3.5607e-04,\n",
            "         2.2233e-03, 7.2173e-04, 9.4996e-01],\n",
            "        [1.9433e-03, 2.0796e-04, 2.9782e-04, 1.8660e-04, 9.3785e-04, 7.0162e-04,\n",
            "         4.5767e-05, 1.3233e-04, 9.4669e-04, 9.9391e-04, 1.2740e-03, 2.9165e-04,\n",
            "         3.2104e-04, 1.1219e-04, 1.4303e-03, 4.0881e-04, 4.3554e-04, 3.0561e-04,\n",
            "         3.0248e-03, 1.0183e-03, 9.8498e-01],\n",
            "        [3.3779e-03, 1.0912e-03, 2.8872e-03, 4.9646e-03, 3.9861e-03, 1.6522e-03,\n",
            "         1.3753e-04, 9.0277e-04, 2.8146e-03, 2.5786e-03, 4.6546e-03, 7.1394e-04,\n",
            "         1.2131e-03, 3.8763e-04, 1.9615e-03, 9.1990e-04, 7.3386e-03, 2.6883e-04,\n",
            "         5.7540e-03, 4.8078e-03, 9.4759e-01],\n",
            "        [1.2428e-02, 2.9023e-03, 1.3185e-02, 1.0254e-01, 1.0864e-02, 1.4462e-03,\n",
            "         5.1316e-04, 1.9269e-03, 4.8718e-03, 1.1832e-03, 7.6414e-03, 5.2390e-03,\n",
            "         9.4042e-03, 1.3550e-02, 8.5623e-03, 7.1719e-04, 2.2186e-03, 2.0015e-03,\n",
            "         1.3174e-02, 1.4595e-02, 7.7103e-01],\n",
            "        [2.3358e-03, 1.6854e-03, 1.0729e-03, 1.3240e-03, 1.3191e-02, 6.1049e-03,\n",
            "         3.9797e-03, 8.5126e-03, 5.8485e-03, 9.8617e-04, 1.3298e-03, 7.5258e-04,\n",
            "         3.8336e-04, 1.0936e-02, 5.3171e-01, 1.4935e-02, 1.6252e-02, 1.8874e-02,\n",
            "         9.1211e-03, 3.5254e-02, 3.1541e-01],\n",
            "        [5.2774e-03, 2.5083e-04, 8.1405e-04, 1.3703e-03, 2.9297e-03, 2.9779e-04,\n",
            "         9.4886e-05, 2.8338e-04, 7.6016e-04, 3.0721e-04, 2.2998e-04, 3.0715e-05,\n",
            "         2.4555e-04, 1.5048e-03, 2.3879e-02, 8.4886e-01, 8.8061e-03, 6.5786e-03,\n",
            "         1.3249e-03, 4.9083e-03, 9.1241e-02],\n",
            "        [5.5073e-03, 3.1287e-03, 1.3831e-03, 1.6199e-03, 2.3198e-03, 1.8324e-03,\n",
            "         5.2003e-04, 6.1077e-04, 2.2793e-03, 2.0206e-03, 3.9681e-03, 1.9237e-03,\n",
            "         5.0447e-04, 1.1097e-03, 1.1338e-02, 1.1839e-02, 1.8599e-01, 4.5732e-03,\n",
            "         1.8949e-01, 1.4003e-01, 4.2802e-01],\n",
            "        [1.0082e-02, 1.5095e-03, 4.3108e-04, 5.6567e-03, 3.6624e-03, 6.3636e-04,\n",
            "         2.5569e-03, 1.9218e-03, 9.1277e-03, 6.8546e-04, 8.0584e-04, 2.5536e-03,\n",
            "         5.9570e-04, 5.0225e-04, 8.4245e-03, 2.1868e-02, 5.7483e-02, 6.1012e-02,\n",
            "         8.4191e-03, 3.7463e-02, 7.6460e-01],\n",
            "        [2.8550e-03, 5.2784e-04, 5.1637e-04, 1.3975e-03, 1.4411e-02, 1.7217e-03,\n",
            "         2.8093e-05, 1.4703e-04, 4.3325e-03, 3.3282e-04, 2.8929e-04, 7.3016e-04,\n",
            "         5.5772e-05, 8.8850e-04, 2.0921e-02, 1.0223e-03, 1.5564e-02, 3.5137e-03,\n",
            "         1.0650e-01, 1.7037e-01, 6.5388e-01],\n",
            "        [1.2881e-03, 2.6872e-04, 4.9264e-04, 1.9707e-03, 4.9985e-03, 3.3745e-04,\n",
            "         2.3043e-04, 1.3519e-04, 4.1040e-04, 1.1987e-04, 8.7405e-04, 6.3427e-04,\n",
            "         1.5307e-04, 2.6046e-03, 1.3134e-03, 1.1503e-03, 7.1137e-03, 2.1219e-03,\n",
            "         4.1314e-02, 9.8485e-02, 8.3398e-01],\n",
            "        [2.9651e-03, 9.9829e-04, 1.5366e-03, 1.3104e-03, 1.9285e-02, 2.8804e-04,\n",
            "         3.3009e-04, 5.3331e-04, 4.9414e-03, 2.0100e-04, 6.6636e-04, 3.3935e-03,\n",
            "         3.0528e-04, 2.1544e-03, 6.0411e-03, 2.6548e-03, 6.3901e-03, 1.3359e-02,\n",
            "         5.4090e-02, 5.2220e-02, 8.2634e-01],\n",
            "        [9.7131e-04, 3.2501e-03, 1.7187e-03, 1.0668e-03, 1.9529e-03, 9.9618e-04,\n",
            "         2.6959e-03, 5.7405e-04, 1.6364e-03, 1.7031e-03, 2.0752e-03, 1.7407e-03,\n",
            "         2.3899e-03, 3.8022e-03, 6.1156e-03, 2.1006e-03, 1.6711e-02, 4.9112e-03,\n",
            "         5.9492e-01, 3.7743e-02, 3.1093e-01]])\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.image.AxesImage at 0x7cf353d1a0b0>"
            ]
          },
          "metadata": {},
          "execution_count": 70
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 420x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAGkCAYAAADOn7VCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAanklEQVR4nO3db2xU973n8c/xv+FP7CEO2GMXQ0z+0Sbg7pLgWEm6pFjYXokNCbcClAcGofQqNZGIFaHLVfkTNZKVdLdFqVx40kKRFpLwAKJEFd3UCeZWBXJDLsqNVFHgOsJcsCns2mM78dieOfsgydw7wQHPOWPOzNfvl3QEnjk//77++czHR2eOv3Zc13UFADAhL+gCAACZQ6gDgCGEOgAYQqgDgCGEOgAYQqgDgCGEOgAYQqgDgCGEOgAYQqgDgCE5F+rt7e26++67NW3aNNXW1urDDz8MuqSstXPnTjmOk7ItXLgw6LKyzvHjx7Vy5UpVVlbKcRwdOXIk5XnXdbV9+3ZVVFRo+vTpqq+v17lz54IpNkvcas3Wr19/w7HX2NgYTLFZoq2tTY888oiKi4tVVlamVatW6ezZsyn7DA8Pq6WlRXfddZfuuOMOrV69Wr29vWnNk1Oh/uabb6q1tVU7duzQxx9/rJqaGjU0NOjq1atBl5a1HnzwQV25ciW5/elPfwq6pKwzNDSkmpoatbe3j/v8a6+9ptdff1179uzRqVOnNHPmTDU0NGh4ePg2V5o9brVmktTY2Jhy7B08ePA2Vph9Ojs71dLSopMnT+q9997T6OioVqxYoaGhoeQ+L774ot555x0dOnRInZ2dunz5sp555pn0JnJzyNKlS92Wlpbkx/F43K2srHTb2toCrCp77dixw62pqQm6jJwiyT18+HDy40Qi4UYiEffnP/958rG+vj43FAq5Bw8eDKDC7PPNNXNd121ubnafeuqpQOrJFVevXnUluZ2dna7rfnlcFRYWuocOHUru85e//MWV5J44cWLCnzdnztRHRkZ0+vRp1dfXJx/Ly8tTfX29Tpw4EWBl2e3cuXOqrKzUggUL9Oyzz+rixYtBl5RTurq61NPTk3LchcNh1dbWctzdwrFjx1RWVqYHHnhAzz//vK5fvx50SVmlv79fklRaWipJOn36tEZHR1OOtYULF2revHlpHWs5E+rXrl1TPB5XeXl5yuPl5eXq6ekJqKrsVltbq3379uno0aPavXu3urq69MQTT2hgYCDo0nLG18cWx116GhsbtX//fnV0dOjVV19VZ2enmpqaFI/Hgy4tKyQSCW3evFmPPfaYHnroIUlfHmtFRUWaNWtWyr7pHmsFmSwU2aWpqSn5/8WLF6u2tlbz58/XW2+9pY0bNwZYGaxbu3Zt8v+LFi3S4sWLdc899+jYsWNavnx5gJVlh5aWFn366aeT8h5Xzpypz549W/n5+Te8E9zb26tIJBJQVbll1qxZuv/++3X+/PmgS8kZXx9bHHf+LFiwQLNnz+bYk7Rp0ya9++67+uCDDzR37tzk45FIRCMjI+rr60vZP91jLWdCvaioSEuWLFFHR0fysUQioY6ODtXV1QVYWe4YHBzUhQsXVFFREXQpOaO6ulqRSCTluItGozp16hTHXRouXbqk69evT+ljz3Vdbdq0SYcPH9b777+v6urqlOeXLFmiwsLClGPt7NmzunjxYlrHWk5dfmltbVVzc7MefvhhLV26VLt27dLQ0JA2bNgQdGlZ6aWXXtLKlSs1f/58Xb58WTt27FB+fr7WrVsXdGlZZXBwMOUMsqurS2fOnFFpaanmzZunzZs365VXXtF9992n6upqbdu2TZWVlVq1alVwRQfsZmtWWlqql19+WatXr1YkEtGFCxe0ZcsW3XvvvWpoaAiw6mC1tLTowIEDevvtt1VcXJy8Th4OhzV9+nSFw2Ft3LhRra2tKi0tVUlJiV544QXV1dXp0UcfnfhEmb5NZ7L96le/cufNm+cWFRW5S5cudU+ePBl0SVlrzZo1bkVFhVtUVOR+5zvfcdesWeOeP38+6LKyzgcffOBKumFrbm52XffL2xq3bdvmlpeXu6FQyF2+fLl79uzZYIsO2M3W7PPPP3dXrFjhzpkzxy0sLHTnz5/vPvfcc25PT0/QZQdqvPWS5O7duze5zxdffOH+5Cc/ce+88053xowZ7tNPP+1euXIlrXmcryYDABiQM9fUAQC3RqgDgCGEOgAYQqgDgCGEOgAYQqgDgCE5F+qxWEw7d+5ULBYLupScwrqljzXzhnVLXybXLOfuU49GowqHw+rv71dJSUnQ5eQM1i19rJk3rFv6MrlmOXemDgD4doQ6ABiSdQ29EomELl++rOLiYjmOc8Pz0Wg05V9MDOuWPtbMG9YtfbdaM9d1NTAwoMrKSuXl3fxcPOuuqV+6dElVVVVBlwEAWae7uzulB/t4su5Mvbi4WJL0uP67ClQYcDUAkBmH//qvnsdGBxOa/18/S+bjzWRdqH99yaVAhSpwCHUANpQU+38Lc7xL0t/EG6UAYMikhXp7e7vuvvtuTZs2TbW1tfrwww8nayoAwFcmJdTffPNNtba2aseOHfr4449VU1OjhoYGXb16dTKmAwB8ZVJC/Re/+IWee+45bdiwQd/73ve0Z88ezZgxQ7/97W8nYzoAwFcyHuojIyM6ffq06uvr/2OSvDzV19frxIkTN+wfi8UUjUZTNgCANxkP9WvXrikej6u8vDzl8fLy8uRfz/7P2traFA6Hkxv3qAOAd4Hf/bJ161b19/cnt+7u7qBLAoCclfH71GfPnq38/Hz19vamPN7b26tIJHLD/qFQSKFQKNNlAMCUlPEz9aKiIi1ZskQdHR3JxxKJhDo6OlRXV5fp6QAA/8mk/EZpa2urmpub9fDDD2vp0qXatWuXhoaGtGHDhsmYDgDwlUkJ9TVr1uhvf/ubtm/frp6eHn3/+9/X0aNHb3jzFACQWVnXpfHrvwCyTE/R+wWAGX+4fMbz2OhAQnfe/28T+stIWdfQCwAs+jwx4mNsYsL7Bn5LIwAgcwh1ADCEUAcAQwh1ADCEUAcAQwh1ADCEUAcAQwh1ADCEUAcAQwh1ADCEUAcAQwh1ADCEUAcAQwh1ADCEUAcAQ+innk3y8oObOxEPbm5gCvi/PvqpD9BPHQCmJkIdAAwh1AHAEEIdAAwh1AHAEEIdAAwh1AHAEEIdAAwh1AHAEEIdAAwh1AHAEEIdAAwh1AHAEEIdAAyh9W4WcQr9fTvcWCxDlQDIND+NtdMZy5k6ABhCqAOAIYQ6ABhCqAOAIYQ6ABhCqAOAIYQ6ABhCqAOAIYQ6ABhCqAOAIYQ6ABhCqAOAIYQ6ABhCqAOAIYQ6ABhCP/Us4o6OBTe54/gb77qZqQMwKn6bxnKmDgCGEOoAYAihDgCGEOoAYAihDgCGEOoAYAihDgCGEOoAYAihDgCGEOoAYAihDgCGEOoAYAihDgCGEOoAYEjWtt51QiE5TqGnse7IiPd5i4o8j5UkNxbzPLagbLavuePXrnsf7Pj7+Z43c7rnsf+v6bu+5i5545T3wT6/bqfQ+0vIz7EiSXnFxd7n9vEakXzWHmSb5wDnbmzf4nlsPDYs6R8ntC9n6gBgCKEOAIYQ6gBgSMZDfefOnXIcJ2VbuHBhpqcBAIxjUt4offDBB/XHP/7xPyYpyNr3YwHAlElJ24KCAkUikcn41ACAm5iUa+rnzp1TZWWlFixYoGeffVYXL1781n1jsZii0WjKBgDwJuOhXltbq3379uno0aPavXu3urq69MQTT2hgYGDc/dva2hQOh5NbVVVVpksCgCkj46He1NSkH/3oR1q8eLEaGhr0+9//Xn19fXrrrbfG3X/r1q3q7+9Pbt3d3ZkuCQCmjEl/B3PWrFm6//77df78+XGfD4VCCoVCk10GAEwJk36f+uDgoC5cuKCKiorJngoApryMh/pLL72kzs5OffbZZ/rzn/+sp59+Wvn5+Vq3bl2mpwIAfEPGL79cunRJ69at0/Xr1zVnzhw9/vjjOnnypObMmZPpqQAA35DxUH/jjTcy/SkBABNE7xcAMMRxXT/NiTMvGo0qHA5rmZ5Sgcd+6jkrLz+4uRPx4Ob22ePayfe+bm48wK87yJee32MtyOPFjwD7qf+s6589jx0aSKh+Ubf6+/tVUlJy0305UwcAQwh1ADCEUAcAQwh1ADCEUAcAQwh1ADCEUAcAQwh1ADCEUAcAQwh1ADCEUAcAQwh1ADCEUAcAQwh1ADBk0v/wtGeO479NZhD8tFN1E8HNHSSfdbtjY57HOgX+XgKBtu71I1db5/oV4GtkVt6I57EFeRPPBs7UAcAQQh0ADCHUAcAQQh0ADCHUAcAQQh0ADCHUAcAQQh0ADCHUAcAQQh0ADCHUAcAQQh0ADCHUAcAQQh0ADCHUAcCQLO6nnvfldrunzc/3Nd4dG/U8Ni8U8jm3977ibsJnn2kfveDzZszwNXXi8889j3WKinzNrVEfa+63F7uf/vu52ns/h/3vvqWex8YGRyX9fkL7cqYOAIYQ6gBgCKEOAIYQ6gBgCKEOAIYQ6gBgCKEOAIYQ6gBgCKEOAIYQ6gBgCKEOAIYQ6gBgCKEOAIYQ6gBgSPa23k3EvbfezfPePteZ5rP97cCI57GJWMzX3H5a2Do+W7EmvvjCx9hhX3P7aVns+Gx3rISf9rc+xkqSU+h9ar9tfxM+xjuOv7lztG3wv/RVeR47OjTxXOFMHQAMIdQBwBBCHQAMIdQBwBBCHQAMIdQBwBBCHQAMIdQBwBBCHQAMIdQBwBBCHQAMIdQBwBBCHQAMIdQBwBBCHQAMyd5+6o7jve+yjz7V7oj3fuiSfPWKdgq898eW/PUld/KC7HHtr6+4rz70I6O+5vbdE92Hgsoyz2PjV6/5mtv183XnaD90vx4o7vU8NuaM6v9McF/O1AHAEEIdAAwh1AHAkLRD/fjx41q5cqUqKyvlOI6OHDmS8rzrutq+fbsqKio0ffp01dfX69y5c5mqFwBwE2mH+tDQkGpqatTe3j7u86+99ppef/117dmzR6dOndLMmTPV0NCg4WF/f1wYAHBrad/90tTUpKampnGfc11Xu3bt0k9/+lM99dRTkqT9+/ervLxcR44c0dq1a/1VCwC4qYxeU+/q6lJPT4/q6+uTj4XDYdXW1urEiRPjjonFYopGoykbAMCbjIZ6T0+PJKm8vDzl8fLy8uRz39TW1qZwOJzcqqqqMlkSAEwpgd/9snXrVvX39ye37u7uoEsCgJyV0VCPRCKSpN7e1N+c6u3tTT73TaFQSCUlJSkbAMCbjIZ6dXW1IpGIOjo6ko9Fo1GdOnVKdXV1mZwKADCOtO9+GRwc1Pnz55Mfd3V16cyZMyotLdW8efO0efNmvfLKK7rvvvtUXV2tbdu2qbKyUqtWrcpk3QCAcaQd6h999JGefPLJ5Metra2SpObmZu3bt09btmzR0NCQfvzjH6uvr0+PP/64jh49qmnTpmWuagDAuBzXza6WadFoVOFwWMucVSpw/HUt9MIpKvI13k+XR79dGt143PvcPrs0umNj3gf76Gzpm+PzCmSgXRorPI/13aVxzEd3y+yKnNtm8cfej/PY4Kjanzii/v7+W77vmMWtd/O8v+AS3sPN9dPGVVJecbHnsYnBQV9z+3mx+M4mP8Ec5Ivc9X6sBG3s3y8HXULu8XsC4eNY/dGdH3oeO1SQ0Pi/w3+jwG9pBABkDqEOAIYQ6gBgCKEOAIYQ6gBgCKEOAIYQ6gBgCKEOAIYQ6gBgCKEOAIYQ6gBgCKEOAIYQ6gBgCKEOAIYQ6gBgSPb2U/cjwN7eiYEB74P99noOsqe5n/EB9rjGFBPgsRLJ9/63GgbyJ/4HDzhTBwBDCHUAMIRQBwBDCHUAMIRQBwBDCHUAMIRQBwBDCHUAMIRQBwBDCHUAMIRQBwBDCHUAMIRQBwBDCHUAMCR7W+8m4pIzxX7mTNUWslP168aUUp4f8jx2Oq13AWBqItQBwBBCHQAMIdQBwBBCHQAMIdQBwBBCHQAMIdQBwBBCHQAMIdQBwBBCHQAMIdQBwBBCHQAMIdQBwBBCHQAMyd5+6rnKcYKbm77k6fP7/WLN0xfkmgf4+rw8FvM8dmCMfuoAMCUR6gBgCKEOAIYQ6gBgCKEOAIYQ6gBgCKEOAIYQ6gBgCKEOAIYQ6gBgCKEOAIYQ6gBgCKEOAIYQ6gBgSNa23nUKCuQ43spzx8Z8zetH3t1VnsfG/+2ir7nlxv2ND0out7/1U7vj75zKyfM+t5vwuWYJH8dagN8vJz8/sLn/19XlnseODI5KemtC+3KmDgCGEOoAYAihDgCGpB3qx48f18qVK1VZWSnHcXTkyJGU59evXy/HcVK2xsbGTNULALiJtEN9aGhINTU1am9v/9Z9GhsbdeXKleR28OBBX0UCACYm7Vs9mpqa1NTUdNN9QqGQIpGI56IAAN5MyjX1Y8eOqaysTA888ICef/55Xb9+/Vv3jcViikajKRsAwJuMh3pjY6P279+vjo4Ovfrqq+rs7FRTU5Pi8fHva21ra1M4HE5uVVXe7/MGgKku4798tHbt2uT/Fy1apMWLF+uee+7RsWPHtHz5jTffb926Va2trcmPo9EowQ4AHk36LY0LFizQ7Nmzdf78+XGfD4VCKikpSdkAAN5MeqhfunRJ169fV0VFxWRPBQBTXtqXXwYHB1POuru6unTmzBmVlpaqtLRUL7/8slavXq1IJKILFy5oy5Ytuvfee9XQ0JDRwgEAN0o71D/66CM9+eSTyY+/vh7e3Nys3bt365NPPtHvfvc79fX1qbKyUitWrNDPfvYzhUKhzFUNABhX2qG+bNkyuTfpsvaHP/zBV0EAAO/o/QIAhmRtP3V3bEyu3z7bHuf1I36+y/NYv73cXff2r1eSn97gbiJzdaQ9d4C92H32v3flvTf4X9uX+Jr7gRc+9j7YZx95P9xv+X2ZicqbOcPz2FO/fsjz2PjIsOinDgBTEKEOAIYQ6gBgCKEOAIYQ6gBgCKEOAIYQ6gBgCKEOAIYQ6gBgCKEOAIYQ6gBgCKEOAIYQ6gBgCKEOAIZkbetdOc6XmxdBtlP1wW/b30D5bCMLDxLe13zhi5/4mvrv/vWS57FvPfgdX3P7atXsMxsSg4Oex5Z98O+ex44lYhPelzN1ADCEUAcAQwh1ADCEUAcAQwh1ADCEUAcAQwh1ADCEUAcAQwh1ADCEUAcAQwh1ADCEUAcAQwh1ADCEUAcAQwh1ADAke/upA7eD1579X8vR3v2JkVFf4w8tmut97scW+Zo7709nvM/93/6Lr7kL+oY9j+36H3d6HhuPDUuvTmxfztQBwBBCHQAMIdQBwBBCHQAMIdQBwBBCHQAMIdQBwBBCHQAMIdQBwBBCHQAMIdQBwBBCHQAMIdQBwBBCHQAMyd7Wu07el5sXbjyztaTDTyvXHG3j6ltevr/xCR/f76m65n7le/+eFf7lor+5Z8/2PvafPvE398wZnoc+sfIzz2NHBkd0nta7ADD1EOoAYAihDgCGEOoAYAihDgCGEOoAYAihDgCGEOoAYAihDgCGEOoAYAihDgCGEOoAYAihDgCGEOoAYAihDgCGZG8/9UTcez/1IPnoz+0U+Pt2uGNjvsYHxk0EXcGU4xT6PNZiMc9jE3F/f+/Az3Gef98CX3OPzS72PPaj3i88j41/PvH1zsHUBAB8G0IdAAwh1AHAkLRCva2tTY888oiKi4tVVlamVatW6ezZsyn7DA8Pq6WlRXfddZfuuOMOrV69Wr29vRktGgAwvrRCvbOzUy0tLTp58qTee+89jY6OasWKFRoaGkru8+KLL+qdd97RoUOH1NnZqcuXL+uZZ57JeOEAgBul9Rb40aNHUz7et2+fysrKdPr0af3gBz9Qf3+/fvOb3+jAgQP64Q9/KEnau3evvvvd7+rkyZN69NFHM1c5AOAGvq6p9/f3S5JKS0slSadPn9bo6Kjq6+uT+yxcuFDz5s3TiRMnxv0csVhM0Wg0ZQMAeOM51BOJhDZv3qzHHntMDz30kCSpp6dHRUVFmjVrVsq+5eXl6unpGffztLW1KRwOJ7eqqiqvJQHAlOc51FtaWvTpp5/qjTfe8FXA1q1b1d/fn9y6u7t9fT4AmMo8/VrZpk2b9O677+r48eOaO3du8vFIJKKRkRH19fWlnK339vYqEomM+7lCoZBCoZCXMgAA35DWmbrrutq0aZMOHz6s999/X9XV1SnPL1myRIWFhero6Eg+dvbsWV28eFF1dXWZqRgA8K3SOlNvaWnRgQMH9Pbbb6u4uDh5nTwcDmv69OkKh8PauHGjWltbVVpaqpKSEr3wwguqq6vjzhcAuA3SCvXdu3dLkpYtW5by+N69e7V+/XpJ0i9/+Uvl5eVp9erVisViamho0K9//euMFAsAuLm0Qt2dQAfCadOmqb29Xe3t7Z6LAgB4k72td6cg12dL0pzlo10xvPHTOtf33EEe573XfA0vjHtvE71+wb94Hjs8OKZ/mOC+NPQCAEMIdQAwhFAHAEMIdQAwhFAHAEMIdQAwhFAHAEMIdQAwhFAHAEMIdQAwhFAHAEMIdQAwhFAHAEMIdQAwhFAHAEPop55FnIJCX+Pd0ZEMVXKbOY6/8fRjT18Or7lT4CO2Cv1Fntsf9Tx21M33MXbi682ZOgAYQqgDgCGEOgAYQqgDgCGEOgAYQqgDgCGEOgAYQqgDgCGEOgAYQqgDgCGEOgAYQqgDgCGEOgAYQqgDgCG03s0iOds61y9a504pvltMj416HvvXX83zNfd9f3/B89g3/meD57HxkWFJ709oX87UAcAQQh0ADCHUAcAQQh0ADCHUAcAQQh0ADCHUAcAQQh0ADCHUAcAQQh0ADCHUAcAQQh0ADCHUAcAQQh0ADMm61rvuV21YxzQq0ZEVmCSOv+E+2iU7rr+5Xdd7693E58O+5h5zvbfH/rJ9rsexo1+OdSew7o47kb1uo0uXLqmqqiroMgAg63R3d2vu3Lk33SfrQj2RSOjy5csqLi6W49z4Ez0ajaqqqkrd3d0qKSkJoMLcxLqljzXzhnVL363WzHVdDQwMqLKyUnl5N79qnnWXX/Ly8m75k0iSSkpKOGA8YN3Sx5p5w7ql72ZrFg6HJ/Q5eKMUAAwh1AHAkJwL9VAopB07digUCgVdSk5h3dLHmnnDuqUvk2uWdW+UAgC8y7kzdQDAtyPUAcAQQh0ADCHUAcAQQh0ADCHUAcAQQh0ADCHUAcCQ/w+E2us9jnAWNgAAAABJRU5ErkJggg==\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Positional Encodings"
      ],
      "metadata": {
        "id": "jMpH_nZM150Z"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now let's take a look at embeddings and positional encodings that are used in transformer models. As we can see from the structure of our pretrained model, it uses sinusoidal position embeddings."
      ],
      "metadata": {
        "id": "bFMoRW6HxU0M"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model"
      ],
      "metadata": {
        "id": "yhXRri4myJqH",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "614c0e9a-2fdb-4b55-f3f1-8c7e89b8caab"
      },
      "execution_count": 71,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "MarianMTModel(\n",
              "  (model): MarianModel(\n",
              "    (shared): Embedding(59514, 512, padding_idx=59513)\n",
              "    (encoder): MarianEncoder(\n",
              "      (embed_tokens): Embedding(59514, 512, padding_idx=59513)\n",
              "      (embed_positions): MarianSinusoidalPositionalEmbedding(512, 512)\n",
              "      (layers): ModuleList(\n",
              "        (0-5): 6 x MarianEncoderLayer(\n",
              "          (self_attn): MarianAttention(\n",
              "            (k_proj): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (v_proj): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (q_proj): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
              "          )\n",
              "          (self_attn_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
              "          (activation_fn): SiLU()\n",
              "          (fc1): Linear(in_features=512, out_features=2048, bias=True)\n",
              "          (fc2): Linear(in_features=2048, out_features=512, bias=True)\n",
              "          (final_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (decoder): MarianDecoder(\n",
              "      (embed_tokens): Embedding(59514, 512, padding_idx=59513)\n",
              "      (embed_positions): MarianSinusoidalPositionalEmbedding(512, 512)\n",
              "      (layers): ModuleList(\n",
              "        (0-5): 6 x MarianDecoderLayer(\n",
              "          (self_attn): MarianAttention(\n",
              "            (k_proj): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (v_proj): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (q_proj): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
              "          )\n",
              "          (activation_fn): SiLU()\n",
              "          (self_attn_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
              "          (encoder_attn): MarianAttention(\n",
              "            (k_proj): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (v_proj): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (q_proj): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
              "          )\n",
              "          (encoder_attn_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
              "          (fc1): Linear(in_features=512, out_features=2048, bias=True)\n",
              "          (fc2): Linear(in_features=2048, out_features=512, bias=True)\n",
              "          (final_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (lm_head): Linear(in_features=512, out_features=59514, bias=False)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 71
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Task 4.**\n",
        "\n",
        "*1 point*\n",
        "\n",
        "Check the encoder embeddings and their shape applying embedding layer to ids of input tokens."
      ],
      "metadata": {
        "id": "3KQO7-sMz_uc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "input_ids"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rJ-0l1VrfKP6",
        "outputId": "6c31028c-2cfd-4c32-f635-9331abd74926"
      },
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[ 1951, 12204,   139,  2444,  8447,  5045, 13378,     9,    19,  4096,\n",
              "          1858,   376,  8058,    63,    37,  7443,   918,    10, 11506,   865,\n",
              "             0]])"
            ]
          },
          "metadata": {},
          "execution_count": 72
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "encoder_embedding_layer = model.model.encoder.embed_positions"
      ],
      "metadata": {
        "id": "3mVjGFTbfMQ0"
      },
      "execution_count": 73,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "embeds = encoder_embedding_layer(input_ids.shape)"
      ],
      "metadata": {
        "id": "5WISc81Zf3pd"
      },
      "execution_count": 74,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "embeds"
      ],
      "metadata": {
        "id": "UBIGhErIyVVP",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "54134ff4-0ec4-4c5a-a35b-1837030ecd77"
      },
      "execution_count": 75,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[ 0.0000,  0.0000,  0.0000,  ...,  1.0000,  1.0000,  1.0000],\n",
              "        [ 0.8415,  0.8219,  0.8020,  ...,  1.0000,  1.0000,  1.0000],\n",
              "        [ 0.9093,  0.9364,  0.9581,  ...,  1.0000,  1.0000,  1.0000],\n",
              "        ...,\n",
              "        [-0.7510, -0.9964, -0.8636,  ...,  1.0000,  1.0000,  1.0000],\n",
              "        [ 0.1499, -0.4977, -0.9202,  ...,  1.0000,  1.0000,  1.0000],\n",
              "        [ 0.9129,  0.4293, -0.2359,  ...,  1.0000,  1.0000,  1.0000]])"
            ]
          },
          "metadata": {},
          "execution_count": 75
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "embeds.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s-I44eJBhZAt",
        "outputId": "a5b298e8-9d29-4743-eb2f-3ac0a64a691c"
      },
      "execution_count": 76,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([21, 512])"
            ]
          },
          "metadata": {},
          "execution_count": 76
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import seaborn as sns\n",
        "import numpy as np\n",
        "\n",
        "position_embeddings_array = embeds.detach().numpy()\n",
        "fig, ax = plt.subplots(1, figsize=(6, 6))\n",
        "plt.figure()\n",
        "ax.matshow(position_embeddings_array)\n",
        "ax.set_aspect('auto')\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 545
        },
        "id": "-bAXVdnqhxLG",
        "outputId": "ec8d3805-ca6f-4db9-cddb-0d0a98f991ec"
      },
      "execution_count": 88,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 600x600 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgMAAAH+CAYAAAABPw0NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABIXElEQVR4nO3deZQkZ33m++/7RkRGZFZVZvVa3S21drWEAAksgdxmtemRkBlsMHcO5nLuAcyxjz2Sx1j2+FozNmCP75U9voOxfTT4ztiAOR6Qsa8BG2ONGQGSsSUhCbVWaLSBWq1e1FtlbZkZEe/v/pFZ2VWlaqkbqheI53NOKN4tM998683UE9VZVc7MDBEREaksf6onICIiIqeWwoCIiEjFKQyIiIhUnMKAiIhIxSkMiIiIVJzCgIiISMUpDIiIiFScwoCIiEjFKQyIiIhUnMKAiIhIxZ12YeCmm27inHPOIcsyrrzySr72ta+d6il937n99tt585vfzKZNm3DO8dnPfnZRv5nx/ve/n40bN1Kv19m2bRuPPvroojEHDx7kne98J81mk/Hxcd773vcyPT19Ep/F6e/GG2/kFa94BWNjY6xfv563vOUt7NixY9GYTqfDtddey5o1axgdHeVtb3sbe/fuXTTmqaee4k1vehONRoP169fz7//9v6coipP5VE57H/nIR7j00ktpNps0m022bt3KP/zDPwz7tc4nzu/+7u/inON973vfsE3rvTI++MEP4pxbdFx88cXD/pO6znYaufnmm61Wq9lHP/pRe/jhh+1nf/ZnbXx83Pbu3Xuqp/Z95Qtf+IL9x//4H+1v/uZvDLDPfOYzi/p/93d/11qtln32s5+1+++/337iJ37Czj33XJubmxuOeeMb32iXXXaZ3XnnnfZP//RPdsEFF9g73vGOk/xMTm9XX321fexjH7OHHnrItm/fbj/+4z9uZ511lk1PTw/H/PzP/7xt3rzZbr31Vrvnnnvsh3/4h+1HfuRHhv1FUdhLXvIS27Ztm9133332hS98wdauXWs33HDDqXhKp62//du/tb//+7+3b33rW7Zjxw77D//hP1iSJPbQQw+Zmdb5RPna175m55xzjl166aX2S7/0S8N2rffK+MAHPmAvfvGLbffu3cPj2WefHfafzHU+rcLAK1/5Srv22muH9bIsbdOmTXbjjTeewll9f1saBkIItmHDBvv93//9Ydvhw4ctTVP71Kc+ZWZmjzzyiAF29913D8f8wz/8gznnbNeuXSdt7t9v9u3bZ4DddtttZtZf1yRJ7K/+6q+GY77xjW8YYHfccYeZ9YOb99727NkzHPORj3zEms2mdbvdk/sEvs+sWrXK/vRP/1TrfIJMTU3ZhRdeaF/84hftda973TAMaL1Xzgc+8AG77LLLlu072et82vwzQa/X495772Xbtm3DNu8927Zt44477jiFM/vB8uSTT7Jnz55F69xqtbjyyiuH63zHHXcwPj7OFVdcMRyzbds2vPfcddddJ33O3y8mJycBWL16NQD33nsveZ4vWuuLL76Ys846a9Fav/SlL2ViYmI45uqrr6bdbvPwww+fxNl//yjLkptvvpmZmRm2bt2qdT5Brr32Wt70pjctWlfQvl5pjz76KJs2beK8887jne98J0899RRw8tc5XoHnsiL2799PWZaLnhTAxMQE3/zmN0/RrH7w7NmzB2DZdZ7v27NnD+vXr1/UH8cxq1evHo6RxUIIvO997+NVr3oVL3nJS4D+OtZqNcbHxxeNXbrWy30t5vvkiAcffJCtW7fS6XQYHR3lM5/5DJdccgnbt2/XOq+wm2++ma9//evcfffdz+nTvl45V155JR//+Me56KKL2L17N7/1W7/Fa17zGh566KGTvs6nTRgQ+X527bXX8tBDD/HVr371VE/lB9ZFF13E9u3bmZyc5K//+q9517vexW233Xaqp/UDZ+fOnfzSL/0SX/ziF8my7FRP5wfaNddcMyxfeumlXHnllZx99tl8+tOfpl6vn9S5nDb/TLB27VqiKHrOJyX37t3Lhg0bTtGsfvDMr+XzrfOGDRvYt2/fov6iKDh48KC+Fsu47rrr+PznP8+Xv/xlzjzzzGH7hg0b6PV6HD58eNH4pWu93Ndivk+OqNVqXHDBBVx++eXceOONXHbZZfzhH/6h1nmF3Xvvvezbt48f+qEfIo5j4jjmtttu44/+6I+I45iJiQmt9wkyPj7Oli1beOyxx076vj5twkCtVuPyyy/n1ltvHbaFELj11lvZunXrKZzZD5Zzzz2XDRs2LFrndrvNXXfdNVznrVu3cvjwYe69997hmC996UuEELjyyitP+pxPV2bGddddx2c+8xm+9KUvce655y7qv/zyy0mSZNFa79ixg6eeemrRWj/44IOLwtcXv/hFms0ml1xyycl5It+nQgh0u12t8wp7wxvewIMPPsj27duHxxVXXME73/nOYVnrfWJMT0/z+OOPs3HjxpO/r4/7448n0M0332xpmtrHP/5xe+SRR+znfu7nbHx8fNEnJeWFTU1N2X333Wf33XefAfahD33I7rvvPvvOd75jZv0fLRwfH7fPfe5z9sADD9hP/uRPLvujhS9/+cvtrrvusq9+9at24YUX6kcLl/iFX/gFa7Va9pWvfGXRjwbNzs4Ox/z8z/+8nXXWWfalL33J7rnnHtu6datt3bp12D//o0FXXXWVbd++3W655RZbt26dfgRriV//9V+32267zZ588kl74IEH7Nd//dfNOWf/+I//aGZa5xNt4U8TmGm9V8qv/Mqv2Fe+8hV78skn7Z//+Z9t27ZttnbtWtu3b5+Zndx1Pq3CgJnZH//xH9tZZ51ltVrNXvnKV9qdd955qqf0fefLX/6yAc853vWud5lZ/8cLf/M3f9MmJiYsTVN7wxveYDt27Fh0HwcOHLB3vOMdNjo6as1m097znvfY1NTUKXg2p6/l1hiwj33sY8Mxc3Nz9m//7b+1VatWWaPRsLe+9a22e/fuRffz7W9/26655hqr1+u2du1a+5Vf+RXL8/wkP5vT28/8zM/Y2WefbbVazdatW2dveMMbhkHATOt8oi0NA1rvlfH2t7/dNm7caLVazc444wx7+9vfbo899tiw/2SuszMz+66/pyEiIiLf906bzwyIiIjIqaEwICIiUnEKAyIiIhWnMCAiIlJxCgMiIiIVpzAgIiJScadlGOh2u3zwgx+k2+2e6qn8wNNanxxa55NHa33yaK1PnhO91qfl7xlot9u0Wi0mJydpNpunejo/0LTWJ4fW+eTRWp88WuuT50Sv9Wn5nQERERE5eRQGREREKi4+1RNYKoTArl27gP63ReTEml9jrfWJpXU+ebTWJ4/W+uQ51rU2M6ampti0aRPeH/v1/mn3mYGnn36azZs3n+ppiIiIfN/auXMnZ5555jGPP+2+MzA2NgbAmR/8DXyWneLZiMhS9//UR0/1FETkKNrTgbN/6NvD/5ceq9MuDDjnAPBZpjAgchpqjumjRiKnu/n/lx4rvapFREQqTmFARESk4hQGREREKk5hQEREpOJOWBi46aabOOecc8iyjCuvvJKvfe1rJ+qhRERE5HtwQsLAX/7lX3L99dfzgQ98gK9//etcdtllXH311ezbt+9EPJyIiIh8D05IGPjQhz7Ez/7sz/Ke97yHSy65hD/5kz+h0Wjw0Y/q55NFRERONyseBnq9Hvfeey/btm078iDes23bNu64447njO92u7Tb7UWHiIiInDwrHgb2799PWZZMTEwsap+YmGDPnj3PGX/jjTfSarWGh34VsYiIyMl1yn+a4IYbbmBycnJ47Ny581RPSUREpFJW/NcRr127liiK2Lt376L2vXv3smHDhueMT9OUNE1XehoiIiJyjFb8OwO1Wo3LL7+cW2+9ddgWQuDWW29l69atK/1wIiIi8j06IX+o6Prrr+dd73oXV1xxBa985Sv58Ic/zMzMDO95z3tOxMOJiIjI9+CEhIG3v/3tPPvss7z//e9nz549vOxlL+OWW255zocKRURE5NQ7YX/C+LrrruO66647UXcvIiIiK+SU/zSBiIiInFoKAyIiIhWnMCAiIlJxCgMiIiIVd8I+QPi9+tMf/+9Eoyk9i+hYwlSo0wkJ+aCeW0R3UM9DRDfE5BYxV9bohogiRBTmKYInmB+Wy8E5LyMMKIMnmKMMDrP+UdqRchi2Mzwz38+gHPpzNnMw6O/XB09m2L6gzsL6gic+38cyfUvq7oXGHk/b0R7/KNzz3cfxWKn7kZPmul1XkvqcepST+ZzU52SuGJx71FxJ4krG/ByRCySupEaJd4EaAe+MCCNxYXDu368HIiBybkF9UHZuWO73HbmOmR+/XN+Rtuff15HTdZFUm14BIiIiFacwICIiUnEKAyIiIhWnMCAiIlJxCgMiIiIVpzAgIiJScQoDIiIiFacwICIiUnEKAyIiIhWnMCAiIlJxCgMiIiIVpzAgIiJScQoDIiIiFacwICIiUnEKAyIiIhWnMCAiIlJxCgMiIiIVpzAgIiJScQoDIiIiFacwICIiUnEKAyIiIhUXn+oJHM15SYd1aUniImIiIje7qL+0QEFJaUZOSW6B3IxZg45F5ObpWEyPiE5IyInohBq5RXQsYaqs07GYPMTkFtEdnHOL6IWYPEQU5pkrE4rgKQb1InjMHIV5yuAJ5shDv2zmKIPDzBEGh5nDjH5bOFJnvo/58uCJGVhwg/LCdgcLxkD/PodtC9ox99wFXW7cgFs43pZ0LjP+aMMWD1hmDss4tlHH4IXmIyvmng+/nKLuKDNHkUGZQpkZodY/WxogCdRGe9RqBVlSkMYF9SSnHudkUc5I3KMe5dR8wap4lkbUJXMFiSvIfE7mchJXkriC2uA83xZhJC4QYXiMyNmgDJGDBIhcf2fNX+1EOLw7stsiHH7BtVC0ZCf6Za6T/DHs1sjp+kq+P2nnioiIVJzCgIiISMUpDIiIiFScwoCIiEjFKQyIiIhUnMKAiIhIxSkMiIiIVJzCgIiISMUpDIiIiFScwoCIiEjFKQyIiIhUnMKAiIhIxSkMiIiIVJzCgIiISMUpDIiIiFScwoCIiEjFKQyIiIhUnMKAiIhIxSkMiIiIVJzCgIiISMUpDIiIiFRcfKoncDRvvOc9rF5nrGvMMJFN8eLRXayLp1gTTTPuZxnzPcZ8SeYcqfOM+RoeT+KiRfdTWiDQIbeSQCC3QI4xFYyeeXI8HYvILaJjCR1LyC2mExJ6FjEV6nRD0u8fnLshJreIIkTk5pkrE3ohpgieYI7CIorgKYOnsH5bGTz5oM3MEQzCYLxBvy24RWczwAZl+mVg0N6/DcMxgydsbtA5Xz9yu0VtC+9v2LCgb2kZcPP3s0zfc9qcHX3copu45x+wzF0u33mMd/IC85EX1vrkXbgowtVquDTFZSlkKdbICPWEsp5QNCI6qxsUmaOoO7oZHEqhzCCkRpkaIQ1QCyQjOUmtIE36RyPJSaOCRtwjiwpG4i6pL1iTzJD5nNTnZC4ncSWZz0lcQc2VZC7HExjxXRJXEmFEGIkLeIzIBSIMD0QOksHziZzDD/ZFNNhI3rlh3S+4Zorcczfawv5gJf4YNmPkdB0mpxftSBERkYpTGBAREak4hQEREZGKUxgQERGpOIUBERGRilMYEBERqTiFARERkYpTGBAREak4hQEREZGKUxgQERGpOIUBERGRilMYEBERqTiFARERkYpTGBAREak4hQEREZGKUxgQERGpOIUBERGRilMYEBERqTiFARERkYpTGBAREak4hQEREZGKi0/1BI7mjD+OCGubPDu+jqdbjts3X0reDDBWkI12aY3Msa4xw0Q2xUTaZm0yxbp4ivOTfYz5HmO+pOEcifNkLiZxEYmrDe9/fQSlBQJGbiWBHrl1yDF6ZnQNeuaZsoTcIjqW0LGE3GI6IaFnER2rkVvEdJnRCf1x3RCTW0QRInLzFKHfVpinV0YUFlEETxk8hXmCOcrgKc0dKQdHCP0+A8wcITjM5o9+m823AZgDwAwY9PcbwMJ8+UjbcPB823z7sM/Bgq5+ty3oX9znlo6f73fLj3/uwOW63PNVlzz+0fsWD3yevmO9j4qzrZcSHZ7DtWewmVnCocOEXg6hBMA7RxpF1MfGcFkK9Qyrp4R6QllPKBpR/8g8RRbRXVWjTGE2g+nU2JcaIQ1QC7haIE5KklpBs9GhkeSkUUEj7pFFBSNxl9QX1KOchu+R+ZxWNEviSjKfk7iCmivJXI4nUHMliStIXEmNQOICHiNyRoQNyuANosHzTVx/00Q4sP7zG9YBP9g40aA9LFgrv8z1lsdRWnhO+0KR03WanFzacSIiIhWnMCAiIlJxCgMiIiIVpzAgIiJScQoDIiIiFacwICIiUnEKAyIiIhWnMCAiIlJxCgMiIiIVpzAgIiJScQoDIiIiFacwICIiUnEKAyIiIhWnMCAiIlJxCgMiIiIVpzAgIiJScQoDIiIiFacwICIiUnEKAyIiIhWnMCAiIlJxCgMiIiIVF5/qCRyN+9pDpC4hi2NcvY7buJ5yvEE+ntIdH6XbGuPJluNbTSNvGmGsoDba4/z1+5moT7E+nWIiabM2brMmnmbczzLmO4y5ghHvGHUJiYvwOBq+tuixSwsEjEBgNuTk9OjZLLlBxzw5ntw8HYvpWMJUqNOziE5IyAdtuUV0Q//cGZxnipTcPEWIhufCPEXwFBZRBE8ZPL0QUQZPMEdpDjM3qDMsGxCCJ4R+vw36zIDBbWzwfCw4GPT3G/pjAMxsWO63c6S83Bm36DTfZ84Wj3PgbMEgt/A+joEB7ig3WHi/R2866s2Papn7OK45V8TeX+vR3t8k2b+GdL8jO2DUD5akB3PiQ3P4qRlsZpZweBI7dGh4OxfHxHFMkqW4LIMsxeop+ZoRykZMUfcUmaeoO4ososxiygzKFPLM2Ds2itUCLi2JkkBSK0iT/lFPcupxThblrElnqUc9Ul/QiHqkriDzOanPyVxOMqiP+C4JJTVXkriCCCNxJRGGd0ZCIHJGYkbkwA82Q2QQOTe8kooIAHgc0YJN5PGUlP0xbuHmeu41mF+y+UoLzxkTOV27yYmj3SUiIlJxCgMiIiIVpzAgIiJScQoDIiIiFacwICIiUnEKAyIiIhW34mHggx/8IM65RcfFF1+80g8jIiIiK+SE/J6BF7/4xfyv//W/jjxIfNr+OgMREZHKOyH/l47jmA0bNpyIuxYREZEVdkI+M/Doo4+yadMmzjvvPN75znfy1FNPHXVst9ul3W4vOkREROTkWfEwcOWVV/Lxj3+cW265hY985CM8+eSTvOY1r2FqamrZ8TfeeCOtVmt4bN68eaWnJCIiIs9jxcPANddcw7/5N/+GSy+9lKuvvpovfOELHD58mE9/+tPLjr/hhhuYnJwcHjt37lzpKYmIiMjzOOGf7BsfH2fLli089thjy/anaUqapid6GiIiInIUJ/z3DExPT/P444+zcePGE/1QIiIi8l1Y8TDwq7/6q9x22218+9vf5l/+5V9461vfShRFvOMd71jphxIREZEVsOL/TPD000/zjne8gwMHDrBu3Tpe/epXc+edd7Ju3bqVfigRERFZASseBm6++eaVvksRERE5gfS3CURERCpOYUBERKTiFAZEREQqTmFARESk4k7bPyc485YraO53RPvacGiS8luPA5D4iLSe4cdGsbERwvgIvVaN7qqYbmuEJzaPsqMZCGMFyWiP1miHNY0Z1tenmEinWF9rsy6e4sLaHsZ8jzFXMOIdqfMkRCQuwuNIXAREpFFCaYGAEQjkVpJbIKekZz1ygymbITdPx2Jyi+lYQscSehbRCcmwbTrOyIdtEUXw5BbRDTFFiMjNU4SIThlTDPqDOUrzFMFTDusOM0deeswcZfAYYIN2MwjBYzZoc/12MAzAHGZG/0b99vmyLW2DBeeFYxe0uQVfOHfkNuYW3Megz9nCwcuYH7/csGGfLdP33Bss91DL3fR5Pd88Kuqfr/gYz5Ql3+yt44G5s3igfQaPHljHzv2jxM+uIjuwmuyAMf54h/jQHL49i03PYHMdrNslHO4AkwC4OCbZPUKSprh6htVTQlYjNBKKekRZj8gbniJzdFdFlFlEmSaUqdFLjU5qWC3gaoGoVpLUCpqNDllckMU5jbhHFhXUo3xw9GhEPVJX0IpnyVxO4goyn1NzJYkrSCiH5QgjcwXeGQmByBkRhjcjcvNXU0YEJDhyIBpsGj/YbBGOMNgzHk9JSeSWbqznXpf5JZuvtPCcMZHT9ZysDO0kERGRilMYEBERqTiFARERkYpTGBAREak4hQEREZGKUxgQERGpOIUBERGRilMYEBERqTiFARERkYpTGBAREak4hQEREZGKUxgQERGpOIUBERGRilMYEBERqTiFARERkYpTGBAREak4hQEREZGKUxgQERGpOIUBERGRilMYEBERqTiFARERkYpzZmanehILtdttWq0WP37Le3l06izyXSM0dnnWf71Luncad3CSMNkmzM2BGTiHT1Pc6AhubJTu2WvorkrotjzdliNvQq9plGMlUbPH2EiH1SOzvHh8NxtqbdYmU6yJplkTTTPu52j4gjFnZM6TupjUxURucWYqLRAwAoHSjFnLyc3IgY45cvN0LCK3iI4l9AbnqVCnE5Jhe24ReYjpWkweIrohJreIubJGYZ48RBTmKYKnWFAuzRPM0S1iDChDv14Gh1n/KO1IOQzbGZ6Z72e+DAz6md8R8+2D8pH2QX2+vPDMgj6W6VtSdy809njajvb4R+FWauefVq+gE++//Ou/4LLaHjbFKalL6FrOs2WXx/ImD3Y2s31qM988vJ49j6wn2+/JDhjZwUB2sCA53MFPzuKmZwkzs1ini+W9/h37CJfEuFoNl2W4LMXqKVavUTZq9FbXKOqevO4pMygyR5lBmUKZGWUKIQ3YWEFUC8RJQa1WkMYl9SSnHvePLM6pRzmrkxnqUU7mc1Kfk7licO5RcyWJK0lcwYjvkriSGiXeBWoEvDMiDI9RcwEPRK5/dRUBkevvwX59UB60DesLrsXmx8/zy1yneZ5/Xy99n5Jqak8FVm15gsnJSZrN5jHfTrtHRESk4hQGREREKk5hQEREpOIUBkRERCpOYUBERKTiFAZEREQqTmFARESk4hQGREREKk5hQEREpOIUBkRERCpOYUBERKTiFAZEREQqTmFARESk4hQGREREKk5hQEREpOIUBkRERCpOYUBERKTiFAZEREQqTmFARESk4hQGREREKk5hQEREpOKcmdmpnsRC7XabVqvF/h3n0Gl0eLC3itunL+aTD7yCeFdK4xnH6O6Sxu4u8b42HJokTE1j3S4AfmQEPzaKjY0QxkfotWp0V8V0W55ey9FrQt4MRBtnaY12WNOYYX19iol0ivW1NuviKVZH06yJphnzPTZEJanzJEQkLsLjiNziDJVbSSCQW0lugRyjZ0ZukOPoWERunqmQ0bGEjiX0LKITEnKL6VhCbhHd0D9PlRlF8IO2mCL0b1+EiMI8RfAUFtErI8pBvQyeYI7SHGZuUAczRxE8Nmg3gxA8NugzA+b7mC8Dg37olxmU58czv2uGZ7ekvqBtKVu+7JaOX7ozn2+nPm/fUeaxhFuJV8Jp9Wo6MdKDnrnzulxw1j5et+5RXj3yLS5K2qyPGkTOMx06PFOWfHrych5on8GjB9ZxeP8o8bMJ2QFHdsCoHyxJD+XEh+ZwzzyLzXWwbhcriuHjuDjuH1kKaQqrW4SsRmgkFPWIsh6RNzxF5igzKOr9c69phLR/WC3gaoGoVpLUCtKkf2RxwepshiwqqEf54OjRiHqkriDzOanPyVzOWDRHzZUkriChHJYjjMSVRBjeGdl8GYjckSutCIicW1B3eOeG5Xl+wbVZ5JZvP9L2wnt66fuUVEN7KrBqyxNMTk7SbDaP+XbaLSIiIhWnMCAiIlJxCgMiIiIVpzAgIiJScQoDIiIiFacwICIiUnEKAyIiIhWnMCAiIlJxCgMiIiIVpzAgIiJScQoDIiIiFacwICIiUnEKAyIiIhWnMCAiIlJxCgMiIiIVpzAgIiJScQoDIiIiFacwICIiUnEKAyIiIhWnMCAiIlJxCgMiIiIV58zMTvUkFmq327RaLX7v7tfwpvWPc16SkLqEOzsl/zJ7IV89eD4P795IsatB4xnP6DOBxp6cdO807sBhir37wAycw6cpbnQENzZKaI2Qr8rojid0W57pzY5e0yjHSqJmj7GRDqtHZllXn2Z9OsWGWpu1yRRbansY93OM+ZyGg8x5UheTuAiPI3JH8lRpgYARCORWUmLkFsjNyIGpENGxiNwiOpbQG5xzi+lYQick5BYxFTK6g3IeIrohJreIYlAvzJOHiE4ZUwzqRfCU5gnmKEO/bkAZPHnpMXOYOcrBuX+wqByCg/k6DMrAYIeYuX55vn0whmF5Qdt8feHumm9fyJYvuxcau1z9aG2L+pe532W4lXhVnFavrJVz0W99AztzI9MXtjh8fsT0eQUT5xzkVRNP8Jrmt7istodNcQrA3rLLE3mTBzub2T61mUcOTbBvfxOeTcn2e7IDxvhjPZLDHfzkLEzNYLNzWKeLFTnDjeYj/EgDl2W4LMXqKVavUTZqlPWYsu4p6p687umOO8o6lDUoM6NMIaQBqxmkJVEtECcFzZEOWVxQj3PqcU4W59SjweF71KOczOe0ojlSn5O5HjVXkriSxBWDckHiSmqUZK7AOyPC8Bg1F/BANNhyHoiAyLlBud/hXf88X++P7b+vRG7xfvXLXL95XnhPL3yfkh9s7anAqi1PMDk5SbPZPObbaYeIiIhUnMKAiIhIxSkMiIiIVJzCgIiISMUpDIiIiFScwoCIiEjFKQyIiIhUnMKAiIhIxSkMiIiIVJzCgIiISMUpDIiIiFScwoCIiEjFKQyIiIhUnMKAiIhIxSkMiIiIVJzCgIiISMUpDIiIiFScwoCIiEjFKQyIiIhUnMKAiIhIxSkMiIiIVJwzMzvVk1io3W7TarW46N/938y9zPPSC57mX69/gPc0d5K4iEPlLI/kGV+duYh/Png+33xmgrCrTuMZz+gzgfEHDsKBw9jUNGFuDszAR/gsxY2M4MZGKFsjzJw7Sqfl6bUcvSbkzUA5VhKP5YyNzrFmZJZ12TQvGXuGiWSSNfE0436WcT/HmM9pOMicJ3UxqYuJ3OJcVVogYAQCuZWUGLOhpGOQ48jN07GI3CI6ltAbnDuWMFXWh+25RXRD/5yHiG6IyS2isIjpokYRIgrzFMETzA/L5eBs5siDpwyeYI4yOMz6R2lHymYQgseMYZ35PubLwKCfQT/0l5jBGIblI/2L2lgwdqGlu3BB3b3Q2ONpO9rjH4VbqVfHafUq+95c/F92Uu7ZixUF0apV2FkbmLqgyeELImbOLdh0zn5ePfEE1675KhujOomL6FrO3rLLo3mL++fOZvvUmXzr0Hr27W9S/0ZGdsCoHwykBwuSQx2iyRlsZhabncM6XazIh69ll8T4NIU0xWUpVk+xRkpZTyjrMZ01MUXmKepQZo4ygzKFMjPKGoQ0YDUjGstJagVJUpAlBVlcUI9z6nFOFueMRD3qUc54Mkvmc1Kfk7licO5RcyWJK8l8j4SSzOfUKPEuUCPgnRFhJC4QYXggcv0rsAiIXH8P9uuD8qBtWF9wvTY/fp5f5lrO8/z7eun7lPzgaU8FVm15gsnJSZrN5jHfTjtDRESk4hQGREREKk5hQEREpOIUBkRERCpOYUBERKTiFAZEREQq7rjDwO23386b3/xmNm3ahHOOz372s4v6zYz3v//9bNy4kXq9zrZt23j00UdXar4iIiKywo47DMzMzHDZZZdx0003Ldv/n//zf+aP/uiP+JM/+RPuuusuRkZGuPrqq+l0Ot/zZEVERGTlxcd7g2uuuYZrrrlm2T4z48Mf/jC/8Ru/wU/+5E8C8IlPfIKJiQk++9nP8tM//dPf22xFRERkxa3oZwaefPJJ9uzZw7Zt24ZtrVaLK6+8kjvuuGPZ23S7Xdrt9qJDRERETp4VDQN79uwBYGJiYlH7xMTEsG+pG2+8kVarNTw2b968klMSERGRF3DKf5rghhtuYHJycnjs3LnzVE9JRESkUlY0DGzYsAGAvXv3Lmrfu3fvsG+pNE1pNpuLDhERETl5VjQMnHvuuWzYsIFbb7112NZut7nrrrvYunXrSj6UiIiIrJDj/mmC6elpHnvssWH9ySefZPv27axevZqzzjqL973vffzO7/wOF154Ieeeey6/+Zu/yaZNm3jLW96ykvMWERGRFXLcYeCee+7hR3/0R4f166+/HoB3vetdfPzjH+fXfu3XmJmZ4ed+7uc4fPgwr371q7nlllvIsmzlZi0iIiIr5rjDwOtf/3rM7Kj9zjl++7d/m9/+7d/+niYmIiIiJ8cp/2kCERERObUUBkRERCpOYUBERKTiFAZEREQqztnzfRrwFGi327RaLX5s7J3E55/H4UtaHLzE8YY33sebVm3nFekB1kcj5Fayu5xje3c9/zR1EXfsO5ddz6xm1d0Jo7tL6ns6xPva2GSb0J7G8h4ALo5x9Tpu0wTlqga98ZRuK6LbcvRajl7TyJuGjRWko10uWL+fiWyKtek0E0mbtXGbNfE0a/wMY75Hw5WsjiIyF+PxJC56znMqLRAwupaTWyDH6JmRG3TMk+PpWERuER1LmAp1ehbRCQkdq5FbRDck/f7BObeImSIlN08RouG5ME8RPIVFFMFTBk8vRJTBE8xRmsPMDeoQBu3GoBz6/WYMzwxuM79RLDgY9Pcb+mOA4fgj7fM3WtDGMu0L2fJlt3Tscjv3WNuGfcs8/lG4lXilnFavtu/e6gccax6cwj++i/LQIVwcE02sJz9nPYcvqNM+H7a87klev+Zb/EjjUS5IOqzydSLnmQxzPF3AN3sT3Dd7Np978qVMPztCsj8m2+/IDhj1gyXpwR7x4Tnc9Cw2PUN5aBJC2Z+Ac7gowtVquDTFpTWoZ1g9JV/doGzEFA1PkXmKzFHUocwcRQYhhTI1imbAagFXK4lrJUmtoBaXZLWcLC6oxzmNuMeq2hz1qEc9ysl8TuqK/tnnZK7flriCEd8loaTmShJXEGEkrhycAx4jckaCEbn+ldj8u0Xk3PDKLKK/J71zR8pLrtsid2TfLu3rt73wvo6crgV/ELWnAqu2PMHk5ORx/RI/7QYREZGKUxgQERGpOIUBERGRilMYEBERqTiFARERkYpTGBAREak4hQEREZGKUxgQERGpOIUBERGRilMYEBERqTiFARERkYpTGBAREak4hQEREZGKUxgQERGpOIUBERGRilMYEBERqTiFARERkYpTGBAREak4hQEREZGKUxgQERGpOIUBERGRinNmZqd6Egu1221arRbbzvh52H0AF0VEZ2zk6Z/aTPtFORdd8Axv3vAAP9bYwXlJQuoSDpWz7MhT/mX2Qv7fh19NsatB4xnP6DOBkd09avtmcAcOE9pThLk5MMNnGW50BDc2SmiNkK/K6I4ndFuebsuRN6HXNDhjjrGRDqtHZllXn2Z9OsWGWpu1yRTr4zbjfpbNcZuGg8x5UheTuAiPI3KLs1ZuJYFAbiUlRm6B3Iwc6JgjN0/HIqZCRs8iOpaQW0zHEjohIR+2RXRDwmxZoxticosoLCIPEYX54bkIniJEdMqY0jzBHGXotxtQhvk2h5nrt5sbHCxTBswRzMGw3n9uZq5fNjeoD8rzu2t4dkvqS9oXWrozB3V3HGNfcNyivmXudxluJV4xp9Wr7vj8f2/5Qz564NX8/TdeQuP+Omsf7NH45l7KXbuxoiAab2FnbuTpN65m+ryCiXMO8qqJJ3hN81tcVtvDpjgldQldy9lbdvmbqZfw4NSZPHJogn37m/BsSrbfkx0wsoOB7GBBcriDf/pZbHYO63SxIh9sMsBHuCTG1Wq4LINVTaxeo2zUKOsxZd2TNzxF5ikzKDJHWe+/vssUQhqwmkFaEtUCcVJQqxVkSUEWF6zOZsninHo0OHyPepST+ZzU52SuIPU5TT9H4koSV1AbnBNXUqPEu0CNgHdGQqDmAh6IBlvOAxEQObeg7vCDesSRvekXXMPNj1/avpDn+ff10vcp+f7Wngqs2vIEk5OTNJvNY76ddoGIiEjFKQyIiIhUnMKAiIhIxSkMiIiIVJzCgIiISMUpDIiIiFScwoCIiEjFKQyIiIhUnMKAiIhIxSkMiIiIVJzCgIiISMUpDIiIiFScwoCIiEjFKQyIiIhUnMKAiIhIxSkMiIiIVJzCgIiISMUpDIiIiFScwoCIiEjFKQyIiIhUnMKAiIhIxTkzs1M9iYXa7TatVouzf+f/YsNDKeP3PUt4cid+9Tj5ljM48OKMwy8yNl60j2s2PcJVYw9ySVIy6jNmQ49PTZ3FbYe3sH3vGUzvalJ/OmJktzGyOyfbO4vfP0mYbBOmpsA5XK2GbzRwzVFCa4SiVac3ntBtebrjnukzjaJZ4sdyGqNdVjXmWN+YYn02zUStzdpkiovTZxj3c4z5nIYzMufIXETiImIiIrc4c5UWKCgpzcgpyS2Qm5EDucFUSOhYTI+ITkjIieiEGrlFdCzpt1nMVJmRW0Q3xOQWkVtEL8TkIaIwPzx3ioTCPEXwmDkK85TBE8xRmuufg6co+/1hcJg5zBicF5TDoA4waAdg0M+gbsENCm7Yz3DsgjaWaV9q6bgBt3T80t18tN39fLv+aHNYwq3UK+e0egW+sNve9v9wZjzKk/k0/2PyCm5+7HLK7S3WPlgy9vB+7KldhE6H6MLzmL1gDZPnJ7TPCzTObfPDm77Da1o7eEX2FGfHMQ1fY185wxN5xsPdM7hv+mwePryBXfvHKfZnpPsjsv2QHQw0H58hOjyLm5rB5uYIcx2s12O4AZ3DxQm+OYrLMqyeYllKaCSUjYSiHlE0PEXmKDJHd5WjTKHMjFCDkBohDVAL+LQkTkpqtYJWvUM9yanHOVnUP0bjHjVf0PA9Ul/QiLqM+Q6Zz0lcSeZyEldQcyWJK0hcSUQY9JVEGB4jcjYoQ+T6V2cREDk3vFKLcHh3ZE9G9Mt+MCJyi/erX+Yaz3Nse3rpe5V8/2lPBVZteYLJyUmazeYx305feRERkYpTGBAREak4hQEREZGKUxgQERGpOIUBERGRilMYEBERqTiFARERkYpTGBAREak4hQEREZGKUxgQERGpOIUBERGRilMYEBERqTiFARERkYpTGBAREak4hQEREZGKUxgQERGpOIUBERGRilMYEBERqTiFARERkYpTGBAREak4hQEREZGKc2Zmp3oSC7XbbVqtFp9/4Dz+++TV3H/v+ay/G1r/331Yt0u0bh3FljM4eEmdQ5cYa7Yc4JozH+GasQd4SS3H43myKLm7czb/dHgLX997Jod3Nanvihl5xhjdXZDuncU/vpMwO4sVBQAuTfGNBm50hNAapRjP6I3XaJ8d02tC3jKKsQBjOY3RLqtHZllbn2ZDfYoXj+xiXdxm3M8yHs0y5nLGfCBzjsxFJC4iJiJyi7NXaYGCktKMnJLcArkZswY983QtooenYwmdkJAT0Qk1covoWMJUWe+XQ0JuEd0Qk1tEbhFF6NcL88yVCUXwFCGiME8wRxk8hXnKMKibIy8jzBxlcJg5gjkMCMFjBmaD9uAwAwZ1Y748eGKDsdjSdjfsZ2Hbwh24dCzL9C0tA27h+OV29LG2LepfZg7LcCv1CjqtXolHF8Zz3vGyu/nZ1f/CuckoTxXT/GX7Mv7isVfSvX8Vax8oaT5ykPIbj/ZfU5smmDt/DYfPrzF1XiA9d4pXnvEUrxvfwSuy7/DiWp3SAgfCHN8uajzSPYOvT5/Nw4c3snP/OMX+Oun+iPFvBbKDJbWDHaLJOdz0LDYzQ5jrYHkBoQTAJTVcLcFlKS7LsHqKZSmhkVA2Eop6RFn3zK3ylJmjzKBMocyMMoWQGiENUAv4Wkl9pEea5GRJQRYX1OOcRtwji3Lqg6Phe7TiOVKfk7mczOckriRzOYkrqLmSxBXDtggjcYEIw2NEzgZliFz/Kq3m+vtv/l0jwuHdkT0Z4fALrucit3i/+mWu9TwvvKeXvk/J95f2VGDVlieYnJyk2Wwe8+30VRcREak4hQEREZGKUxgQERGpOIUBERGRilMYEBERqTiFARERkYpTGBAREak4hQEREZGKUxgQERGpOIUBERGRilMYEBERqTiFARERkYpTGBAREak4hQEREZGKUxgQERGpOIUBERGRilMYEBERqTiFARERkYpTGBAREak4hQEREZGKUxgQERGpOGdmdqonsVC73abVavHkNzawsVXny3MZH3rqKqb+ZDOt+/YRvv00VuTEE+vpXbSJA5dkHH5RYNOWZ/nxTQ/z71bfz6jPmA4dnijgzrnzuP3QFu7fu4mZXWPUd0WMPGOs2X4Yf6BNmGwTZmYhlOAcrlbDNxq45iihNcLs2U064xG9lqPXhF7TKJolfiynMdplzcgsl6zaw0StzdpkinVxmzXRNON+jjGf03BG5hyZi6i7GpF7bv4qLVBQUpqRUzIbSnIgN+hYRG6ejsX0iOiEhJyITqgxFbJ+3WI6FpOHmNwiuoNzbhG9EDNXJuQhojBPETzFgrKZozBPGTx58ARzlIP2MjjMHGFwmDnMIASPGcM6833Ml/vPy4KD+d21oB1b2D6oz5dhcXm+j2X6lpYBt3D8cjv7WNuGfcs8/jLcSr6KTqtX5HNd9N8OsPe1azn8mg7vufQO3j1+D2fGozyeT/MXh1/Jpx97Oba9xTk378F2PkPodPAjI7gzNjB7/moOX5AwdV5g5NxJrtz4FNdPfJGz45iGr5FbyaHQ4Yk844HuZh6Y3szDhzewa/840TdGyA5A/UAgPVSQHuziJ2dxUzPYzCyh28V6PTDrv5bjBFdLcPUMl2VYVsMaKSFLKBsJnTUJReYo6o4ycxQZlCmUmRFq/bOlAT+aEycltVpBLS6oJwX1JKce52RRzkjcox7ljMezpL6gEXXJXEHiCjKfk7mcxJUkrqDmShq+S0QgcSURRuICEYbHiJwNypA4iIDI9ffg/DtHhMMP2iLm+468r8yPn+eXuebzvPC+Xu69Sk5/7anAqi1PMDk5SbPZPObb6astIiJScQoDIiIiFacwICIiUnEKAyIiIhWnMCAiIlJxCgMiIiIVd9xh4Pbbb+fNb34zmzZtwjnHZz/72UX97373u3HOLTre+MY3rtR8RUREZIUddxiYmZnhsssu46abbjrqmDe+8Y3s3r17eHzqU5/6niYpIiIiJ058vDe45ppruOaaa553TJqmbNiw4buelIiIiJw8J+QzA1/5yldYv349F110Eb/wC7/AgQMHjjq22+3SbrcXHSIiInLyrHgYeOMb38gnPvEJbr31Vn7v936P2267jWuuuYayLJcdf+ONN9JqtYbH5s2bV3pKIiIi8jyO+58JXshP//RPD8svfelLufTSSzn//PP5yle+whve8IbnjL/hhhu4/vrrh/V2u61AICIichKd8B8tPO+881i7di2PPfbYsv1pmtJsNhcdIiIicvKc8DDw9NNPc+DAATZu3HiiH0pERES+C8f9zwTT09OLrvKffPJJtm/fzurVq1m9ejW/9Vu/xdve9jY2bNjA448/zq/92q9xwQUXcPXVV6/oxEVERGRlHHcYuOeee/jRH/3RYX3+3/vf9a538ZGPfIQHHniAP//zP+fw4cNs2rSJq666iv/0n/4TaZqu3KxFRERkxRx3GHj961+PmR21/3/+z//5PU1IRERETi79bQIREZGKUxgQERGpOIUBERGRilMYEBERqThnz/dpwFOg3W7TarW46JP/J++/4ku8dXQfuZX82P3/B1NfW8fEPTkj9z9DsesZXBQRnbGR2YsnOPDiGlMX5/zqq2/hxxo7OC9JSF3CoXKWHXnKv8xeyFcPns/DuzdS7Gqw/h4Y2d2jtm8Gd+AwoT1FmJsDM3AOn6a40RHszAnyVRnd8YRuy9NtOfIm9JpGOVYSNXucvf4g6+rTrE+n2FBrszaZYn3cZtzPMu7nGPM5DQctXyNxER5H5BbnsNICASMQ6FhBboHcjBzomCM3T8cicovoWELPIqZCnY4ldEIybM8tojuo5yGiG2LmQo08RBTmh+cieIoF5dI8eRlRBI8BZfAEc5TBYdY/ysE5LGgzY3hmvo1BOfSfm5mDQX+/PnjSw/YFdRbWFyzQfB/L9C2ou6Xjlht7PG1He/yjcCv1ajqtXpWLXfjr90EweNlF7HrdGO7Vh/ili77M/zb2JKMuZXuv4M8PvIov3/wK1j7Yo/HNvZS7dmNFQTTews7cyPSFLQ6fHzF9XsFPXXkPr2l+i8tqe9gUp6QuoWs5e8suT+RNtnfO4v6pzdz+6AXwbEq235MdMLKDgexgQXK4g5+chakZbHaOMD19ZJP5CJfEuFoNl2W4LMXqKVav0VvToKx78oanyDxlBkXdUWZQplCm1j+PlZCWRLVAnBTUagVZUpDFBfU4px7nZHHO6tosdd+jHuVkPif1OZkrBuceNVeSuJIR3yVxBYkrqVHiXaBGwDsjwvAYNRdIBlvOAxEQObegPig7Nyz3+468r8yPX9q+kOf59/XS9yk5/bWnAqu2PMHk5ORx/RI/faVFREQqTmFARESk4hQGREREKk5hQEREpOIUBkRERCpOYUBERKTiFAZEREQqTmFARESk4hQGREREKk5hQEREpOIUBkRERCpOYUBERKTiFAZEREQqTmFARESk4hQGREREKk5hQEREpOIUBkRERCpOYUBERKTiFAZEREQqTmFARESk4hQGREREKs6ZmZ3qSSzUbrdptVq8dutvsPua1Wx5/RP8xll/x4aoy58dupL/8dArGflanYl7Zokf+Q7loUP4RgN3zplMvngV+97a5dXnPc6/Xn0/V2bPsDFqALC7nOWh3hpun7qIO/afy667NzGyyzG6u6S+p0P87BQcmiRMTWPdLgAujonWrsGao5TjDfLxlO54TLfl6LYcedPIx4x40yyt0TnWNmaYqE+xtjbN+lqbiXiSNfE0436WMd9jQ1SSOk9CROIiPI7ILc5jpQUKSnIryS2QY/TMyA065snx5ObpWMxUqNOxhJ5FdEJCbjEdS8gtohv6505ImCuTQd1ThGh4LsxTBE9hEUXw5KF/LoMnmKM0h5kb1FlQ7reH0D/boM8MGPQZ82Vg0A/9MoPy/Hjmd+Dw7JbUl7QvZMuX3dKxS3f58+365+1bZg7LcCvxqjqtXplHjD3h2fSP+yh3PEa0ahWzP3wBz7w2ZsvWb/PLm/+R12Y9civ56cd/gocePJvV93tWPzRL/OjTlAcOgvPE69dSnDPB5AUN9v2rHls27+W1ax/j1aM7uDiZYY2vEznPdOiwswg8mq/lE3t+hEcPrKO9f4Rkf0K635EdMOoHS9KDOfHkHL49S9i3H+v1sKIYztnFcf/IUlyWQZZSrhkjZAnFSExR9xSZp6g7igzKzFFmUKaQNwMhNawWcGlJlASSWkGa9I96kpNGBWuzGepRTj3qkfqCRtQjdQWZz0l9TuZyElfQjDokriChpOZKElcQYSSuJMLwzkgI1FzAA5E7ctUWAZFzC+oO79ywPM8vuM6L3PLtR9peeE8vfZ+S01d7KrBqyxNMTk7SbDaP+Xb6CouIiFScwoCIiEjFKQyIiIhUnMKAiIhIxSkMiIiIVJzCgIiISMUpDIiIiFScwoCIiEjFKQyIiIhUnMKAiIhIxSkMiIiIVJzCgIiISMUpDIiIiFScwoCIiEjFKQyIiIhUnMKAiIhIxSkMiIiIVJzCgIiISMUpDIiIiFScwoCIiEjFKQyIiIhUnDMzO9WTWKjdbtNqtXg9P0m28UwO/ui57NlWcPdVf8gqX+efOjEffvpf8eC957L+blj19f2EJ57C8h7RunXsfesFHH6Rse6i/VxzxiNcPfYgL6nljPqM2dDjyaLk7s7Z/ME330D7mTEaT8c0dhujuwuyPTP4A23C4UnCzCyEEpem+EYDNzpCaI1SjGf0xmt0xz3dlqfXhLkzSxjLaYx2WT0yy9r6NBvqU6yrTbE+abMubjPuZ9kcTzLmA5lzZC4icRExEZFbnMlKCxSUlGbklOQWyM3IgdygYxG5edqW0gkJORGdUCO3iI4l9CymG5J+PSRMlym5ReQW0QsxeYgozA/PRfAUIaJTxpg5CvOUwRPMUZrrn4PHzFGGQRkIwWMGZm5wMDwzuB3zdYBBP8P6kbHz/Qv7hm2woN8tv3FsyXnALR2/dLcfbfc/36viaHNYwq3UK+u0eoXCL151C3/4T1ex+RYYve1RykOHiC48j2dfM8GB1/b431/2Nd676g4iB5+afDn/4/EryLevYs2DJc2HDmA7nyHMzvZfV5sm2H3VRqbOD2TnTvGKTU/xuvEdvCL7DmfHjlGfUVrgQJjj76bP5/6ZzTx8eCM7949T7K+TPhuRHYDsYCA7WFI72CHedQCbmSHMdbBej+EGdA4XJ7hagstSWNXCspTQSCgbCUU9oqx78rqjyBxl5igz6LWMMoWQGiENUAv4WklcK0mSkjTJyZKCVdkcjbhHFuXUB0fD90h9QeZzUp+TuZyxqEPmchJXUHMliStIXElEGJyNxAUSApEzIgwPRK5/9RYBkevvwX7d4d2RPRkx33fkfSVa0O+Pcg3oeeF9vfS9Sk4/7anAqi1PMDk5SbPZPObb6SsrIiJScQoDIiIiFacwICIiUnEKAyIiIhWnMCAiIlJxCgMiIiIVpzAgIiJScQoDIiIiFacwICIiUnEKAyIiIhWnMCAiIlJxCgMiIiIVpzAgIiJScQoDIiIiFacwICIiUnEKAyIiIhWnMCAiIlJxCgMiIiIVpzAgIiJScQoDIiIiFacwICIiUnHxqZ7A0cy++QrS25+k9Vf30Hz8Rbz97HfwgfP+ltfXA5ec9zd8bPXL+OgZW5k+Yz0T9zSpPfQdymefZeK2Fo1n13Dw2fV84kVj7Dhvgp9Yu50fqe/kjKjBi2s1VvsnePLcddw5di6PNdeRNzPysYSRsTEaozWSekp0qE1ot7FulzIv8L0ertcj6Y4SdRpEnZS4ExN1PCGN6HUdM92IXi9iLk+YLWpM12vMpjVmazVm4ykyn5PTYcwV5K6g4Y0EIyHC44hcP5tFzoOBd0aEI6IkIhBheAxPSe4CZXBEPpBbTOSNnkV4C/gQiHygYwneBQKO3CI6ISEiELuIboj792UeT4R3RsBRBI8zwzujCB5v/TZ8wMzRz4+BMnj8oM0MzIwQPGCAwzCcOXAGDjC3sGfwVTbcoAVzDJsHfX1uSXVQsEWDB4+x4Dx/L24wj6XjjlZf2M7R+o4yhyXmu91y93E8jjbHU+QXx5/g6jd+mH934dvZecGLOPOWg5QP72DtnmdpfmcLf/2d1/DPP3Iet1zy17xv9SO8vP5t/nTta7l7w/nMrl/HmgdHSR59hnLfs/D4t5m4e5Ts0AiTB1t85dBFPHX2Kr6zbi2vHt3BJckk66MG66MRXt94jA3JYTbU2jxQP4NvNdZxqDFG0UgoGp687qjXG4x2C3wS45MaNjuLdbtYUYAZlvewssSXJS6O8XmBy1NcHvB5QpFH+Dwiyj15Ab6AEDtcAWUJrvCE0hEKR156ytITgqMoIxIfKIKniD1FiPpHFFGPegQcpXmC9ySupHSOzDsCnnLQV3MlJZ4aZX+hHWBhsBftyP4GSjMiwDtHOd+3oB7hCIR+G74/3vVvvLB9oYDhef49XVoYvk/JDxZ9VUVERCpOYUBERKTiFAZEREQqTmFARESk4hQGREREKk5hQEREpOIUBkRERCpOYUBERKTiFAZEREQqTmFARESk4hQGREREKk5hQEREpOIUBkRERCpOYUBERKTiFAZEREQqTmFARESk4hQGREREKk5hQEREpOIUBkRERCpOYUBERKTiFAZEREQqLj7VEzia5i88zdNbXsKZn99L+bUHmf7EVt697We47oe+wnvHH+b61d/k5Zd/mw+tu4rH15/F+nUX0NreovzW4zQnp8gObuLAoYw7D23hqS2reGLTw1w99iAXJYGN8Shva93LGbVD3F7fwv0jm2iPjpGPROSNOiMjMVm9RpTWKJ7ZA6EkzM7iigJfFES9nLQ3QtStE3UTynqM73l6PUeeOw7mEZ08plMkzBY1ZsqU2VpKw3fJo4jczzHmc8pQkrlARiBxERhErp/PIueJgNIcOPB4PCWeQIQRYeSuJMLoYHgX8CEhcv3+3EoiC3RCQu4jfDC8N7rEeDO8M3rO8CEiGpyDOTwRhXkI4L31y4A3R2kO5wyIgICZIwABAIf3/TYzMINAAHNgDnOGw2HQvw8Dw9H/r8OwwVfe0R/E4Gz9+3ALNsfC9oWGt1kwDjDXL7j58e5I37DOkrZjsdwclmGuP/R78t3O8QR40443898u+Es+f/Hf8PvrXspHz3kdZ33hChpf3UH8pa9z3jMXsPeZTfz+tS/l3eP3cFUj4fzNn+Mvxl7JX679IXatbbJ27TmMPjyK7XwGHnmCVZMbSA+v5nC7xrfbm3j23BGe3riK149/kx9Kd3J2HHNWXGfMH2CNn2FtMsXa2gwPZRvZlY0zU08psogyi4hnR0hrMb6W4OMIm4kI3S7W6w02ZknoBvz0DJQlrgz4ssSVKS5P8EWCKyNc6SkKR0gcrgRXOsr5c+Ywc5SloxscZeKZiUuK4CnNU1hOYZ6AI+DILaLhe+QWkbiSxEUEPMHllM5TuoKAJ6EAN3jdD8rBjBIjckYwwwPRYD9Ew6+KDU/eOcpBPcIRCP13DzMid2S/zrcvFDA8z7+nSwvD9yn5waGvqIiISMUpDIiIiFScwoCIiEjFKQyIiIhUnMKAiIhIxSkMiIiIVNxxhYEbb7yRV7ziFYyNjbF+/Xre8pa3sGPHjkVjOp0O1157LWvWrGF0dJS3ve1t7N27d0UnLSIiIivnuMLAbbfdxrXXXsudd97JF7/4RfI856qrrmJmZmY45pd/+Zf5u7/7O/7qr/6K2267jWeeeYaf+qmfWvGJi4iIyMo4rl86dMsttyyqf/zjH2f9+vXce++9vPa1r2VycpI/+7M/45Of/CQ/9mM/BsDHPvYxXvSiF3HnnXfywz/8wys3cxEREVkR39NnBiYnJwFYvXo1APfeey95nrNt27bhmIsvvpizzjqLO+64Y9n76Ha7tNvtRYeIiIicPN91GAgh8L73vY9XvepVvOQlLwFgz5491Go1xsfHF42dmJhgz549y97PjTfeSKvVGh6bN2/+bqckIiIi34XvOgxce+21PPTQQ9x8883f0wRuuOEGJicnh8fOnTu/p/sTERGR4/Nd/aGi6667js9//vPcfvvtnHnmmcP2DRs20Ov1OHz48KLvDuzdu5cNGzYse19pmpKm6XczDREREVkBx/WdATPjuuuu4zOf+Qxf+tKXOPfccxf1X3755SRJwq233jps27FjB0899RRbt25dmRmLiIjIijqu7wxce+21fPKTn+Rzn/scY2Njw88BtFot6vU6rVaL9773vVx//fWsXr2aZrPJL/7iL7J161b9JIGIiMhp6rjCwEc+8hEAXv/61y9q/9jHPsa73/1uAP7gD/4A7z1ve9vb6Ha7XH311fzX//pfV2SyIiIisvKOKwyY2QuOybKMm266iZtuuum7npSIiIicPPrbBCIiIhWnMCAiIlJxCgMiIiIV9139noETaf5zCcVsj7LboSi7lJZT9jqEuS6d6YJ2FChcYGaupJjpEjodijwMxxJ6FEWHsgdhLlDMdOlM50wTaCeB4APTvcBcpyCf6VHOdglzCWU3ouwZRZFTlF186FJYDlYCDmcOH3q4EBPKmKJwFHlJ2YspuxA6RkhKiHJK36OgSx569Ioe3VrObFGSRSWxDzgfKJ2RO0fPORLniHFEbvF6lBYoCJRm5AQKC+Rm5MB0cOQGXQv0cHRCSQF0QkkOdM3RCY5OKMiDUVigG4zc+vfTC4E8RBTmKSwiLwPFoF4Gh5kblD3BHKX128oQDfvD4DBzmDE4BuXgYL4PBuX5LzQYDmxp+6BtMGbYtqi+pH3RBjpKGXALxy/38ZdjbVvUv8wclnFso47BC39s54QrZrpMTQXqUaAznRPmBq8/61FajpVdyl6HznTOVBRox4HpPNCdzilnO9CtUeQFRdnFrAcW4couRd6h7JaETqCc7dKb7jEXF0z3+veROMdUCMzkgbleQW8677/+ZzuEORu8fqEoekRlF192caGHWY9gOWY5Rzaaw1sPFzwuOKw0rIRQlJRFSZFHFLGn9I6y5ykdlEDACAGCGRYCFAHKkpCUlK5LERUUSU4eF0RRTi/qEUU5+JLI5+ALOnFB4QrMFwRXklCSuJLClcSuJHGBiP57XL9seIzIGVH/VUPk+ldyNdffWfNXdREO547stogl/W7xTlzuatAfw25d+j4lp4/2dACO7TN+Czk73lucYE8//bR+JbGIiMj3YOfOnYt+KeALOe3CQAiBHTt2cMkll7Bz506azeapntIPtHa7zebNm7XWJ5jW+eTRWp88WuuT51jX2syYmppi06ZNeH/snwQ47f6ZwHvPGWecAUCz2dQGO0m01ieH1vnk0VqfPFrrk+dY1rrVah33/eoDhCIiIhWnMCAiIlJxp2UYSNOUD3zgA/prhieB1vrk0DqfPFrrk0drffKc6LU+7T5AKCIiIifXafmdARERETl5FAZEREQqTmFARESk4hQGREREKk5hQEREpOIUBkRERCpOYUBERKTiFAZEREQq7v8HP/UMWR1VVDkAAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 0 Axes>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Get positional encodings applying its layer to the shape of input tensor."
      ],
      "metadata": {
        "id": "HZeiBURcy6o_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pos_encodings"
      ],
      "metadata": {
        "id": "1QlCCrOPyImm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Visualize obtained positional encodings using heatmap."
      ],
      "metadata": {
        "id": "j7SzzU4czdMq"
      }
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "xp95nwn8zia8"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Training"
      ],
      "metadata": {
        "id": "LvUvNLcD1_Tc"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now we can move on to fine-tuning of the pretrained model on our data. First let's create data collator to form batches with padding to maximum length of sequence within a batch."
      ],
      "metadata": {
        "id": "17LAYH0pzlOt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized_datasets = dataset"
      ],
      "metadata": {
        "id": "l-cH9WuZmtp4"
      },
      "execution_count": 89,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 90,
      "metadata": {
        "id": "X_mfpEeB2qbH"
      },
      "outputs": [],
      "source": [
        "from transformers import DataCollatorForSeq2Seq\n",
        "\n",
        "data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 91,
      "metadata": {
        "id": "8S9l27KL2qbH",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "87fa6f0f-1c8c-48a7-a0b4-3474fb6eb5bd"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_keys(['input_ids', 'attention_mask', 'labels', 'decoder_input_ids'])"
            ]
          },
          "metadata": {},
          "execution_count": 91
        }
      ],
      "source": [
        "batch = data_collator([tokenized_datasets[\"train\"][i] for i in range(0, 3)])\n",
        "batch.keys()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 92,
      "metadata": {
        "id": "8T4Y6w0g2qbI",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "73459cb9-a70a-4630-9c97-2353cee4a612"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[  491, 19219,   954,   376, 24087, 33692,    22,  1084,  6792,  4789,\n",
              "             0,  -100,  -100,  -100,  -100,  -100,  -100,  -100],\n",
              "        [   89,  8283, 11724,  3368,  2009,  2558,  8605,    87,     6, 11699,\n",
              "           150,  2558,  7605, 11396, 18113, 23411,  3495,     0],\n",
              "        [   60, 14800, 15093,  3994,    78,    17,  1833,    20,     6,   112,\n",
              "          6517,    17,  1612, 13147, 12149,     0,  -100,  -100]])"
            ]
          },
          "metadata": {},
          "execution_count": 92
        }
      ],
      "source": [
        "batch[\"labels\"]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 93,
      "metadata": {
        "id": "4gUJcHIO2qbI",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "71444760-faa5-47f0-8995-dd9feadbfe5a"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[59513,   491, 19219,   954,   376, 24087, 33692,    22,  1084,  6792,\n",
              "          4789,     0, 59513, 59513, 59513, 59513, 59513, 59513],\n",
              "        [59513,    89,  8283, 11724,  3368,  2009,  2558,  8605,    87,     6,\n",
              "         11699,   150,  2558,  7605, 11396, 18113, 23411,  3495],\n",
              "        [59513,    60, 14800, 15093,  3994,    78,    17,  1833,    20,     6,\n",
              "           112,  6517,    17,  1612, 13147, 12149,     0, 59513]])"
            ]
          },
          "metadata": {},
          "execution_count": 93
        }
      ],
      "source": [
        "batch[\"decoder_input_ids\"]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 94,
      "metadata": {
        "id": "stdHyAuO2qbJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c6398200-71f2-43d1-bea4-db8178ade9f3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[491, 19219, 954, 376, 24087, 33692, 22, 1084, 6792, 4789, 0]\n",
            "[89, 8283, 11724, 3368, 2009, 2558, 8605, 87, 6, 11699, 150, 2558, 7605, 11396, 18113, 23411, 3495, 0]\n",
            "[60, 14800, 15093, 3994, 78, 17, 1833, 20, 6, 112, 6517, 17, 1612, 13147, 12149, 0]\n"
          ]
        }
      ],
      "source": [
        "for i in range(3):\n",
        "    print(tokenized_datasets[\"train\"][i][\"labels\"])"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We will perform training cycle using `Trainer` object. Trainer requires `TrainingArguments` with all parameters that will be used for training including learning rate, batch size for traning and evaluation, weight decay etc."
      ],
      "metadata": {
        "id": "x8EQ0x2w1rEc"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 95,
      "metadata": {
        "id": "12eMQjMY2qbK"
      },
      "outputs": [],
      "source": [
        "from transformers import Seq2SeqTrainingArguments\n",
        "\n",
        "training_args = Seq2SeqTrainingArguments(\n",
        "    f\"mt-en-fr-med\",\n",
        "    evaluation_strategy=\"no\",\n",
        "    save_strategy=\"epoch\",\n",
        "    learning_rate=2e-5,\n",
        "    per_device_train_batch_size=32,\n",
        "    per_device_eval_batch_size=64,\n",
        "    weight_decay=0.01,\n",
        "    save_total_limit=3,\n",
        "    num_train_epochs=3,\n",
        "    predict_with_generate=True,\n",
        "    fp16=True,\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "During evaluation step on a training process the trainer can compute additional metrics apart from calculating loss. To enable it we will define `compute_metrics` function with BLEU calculation."
      ],
      "metadata": {
        "id": "hrcEYTA358HV"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 96,
      "metadata": {
        "id": "sLJ8B1_32qbJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7134f6fa-f73d-4a04-8022-570002355c69"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting sacrebleu\n",
            "  Downloading sacrebleu-2.4.1-py3-none-any.whl (106 kB)\n",
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/106.6 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m106.6/106.6 kB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting portalocker (from sacrebleu)\n",
            "  Downloading portalocker-2.8.2-py3-none-any.whl (17 kB)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.10/dist-packages (from sacrebleu) (2023.12.25)\n",
            "Requirement already satisfied: tabulate>=0.8.9 in /usr/local/lib/python3.10/dist-packages (from sacrebleu) (0.9.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from sacrebleu) (1.25.2)\n",
            "Collecting colorama (from sacrebleu)\n",
            "  Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.10/dist-packages (from sacrebleu) (4.9.4)\n",
            "Installing collected packages: portalocker, colorama, sacrebleu\n",
            "Successfully installed colorama-0.4.6 portalocker-2.8.2 sacrebleu-2.4.1\n"
          ]
        }
      ],
      "source": [
        "!pip install sacrebleu"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 97,
      "metadata": {
        "id": "mkaJ8MAk2qbJ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "fe1b9fda9a3847a0806158c4689e060f",
            "150e7c620f6746f7bc0ea2b6c94713cb",
            "52c025f2ab3247ffae9233ec215b373e",
            "c95368b3cdc94a3aa032d55a1b76ff78",
            "7e4c1f72cfa8426c95953cdad952966a",
            "18eee73eeb324310a4d14595a84562fa",
            "90e86d07551344daa0db665399af0b12",
            "3dc7bd05088a48e98568c8bdfb836946",
            "33d55f89ebc648a1aa96adf7632cdcd5",
            "25d5ab5774284b5095b9adab13889906",
            "0b7aed03b8c44aa4938771fcdcd6f98e"
          ]
        },
        "outputId": "21c369f6-5781-4cac-b31d-f9b37840d9c6"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading builder script:   0%|          | 0.00/8.15k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "fe1b9fda9a3847a0806158c4689e060f"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "import evaluate\n",
        "metric = evaluate.load(\"sacrebleu\")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's see what the metric returns."
      ],
      "metadata": {
        "id": "HqoIG2SU6oc1"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 98,
      "metadata": {
        "id": "2j32giFK2qbJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "56c14dbc-4b60-404d-f7d6-6c447c174696"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'score': 15.310245441182436,\n",
              " 'counts': [9, 4, 1, 0],\n",
              " 'totals': [15, 14, 13, 12],\n",
              " 'precisions': [60.0,\n",
              "  28.571428571428573,\n",
              "  7.6923076923076925,\n",
              "  4.166666666666667],\n",
              " 'bp': 1.0,\n",
              " 'sys_len': 15,\n",
              " 'ref_len': 14}"
            ]
          },
          "metadata": {},
          "execution_count": 98
        }
      ],
      "source": [
        "predictions = [\n",
        "    \"Clinical performance of eculizumab treatment for thrombotic microangiopathy induced by gemcitabine: a case study\"\n",
        "]\n",
        "references = [\n",
        "    [\n",
        "        \"Clinical efficacy of eculizumab as treatment of gemcitabine-induced thrombotic microangiopathy: a case report\"\n",
        "    ]\n",
        "]\n",
        "metric.compute(predictions=predictions, references=references)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now we can implement `compute_metrics` function. Note that since tokenizer can return special tokens (as `<s>` in the end of sequence), we want to take it into account and ignore these tokens for metrics calculation. Special tokens correspond to -100 in labels so we will replace them with padding token id."
      ],
      "metadata": {
        "id": "a8-tlZiQ7CUN"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 99,
      "metadata": {
        "id": "IMDFnLk62qbK"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "\n",
        "def compute_metrics(eval_preds):\n",
        "    preds, labels = eval_preds\n",
        "    # in case the model returns not only prediction logits\n",
        "    if isinstance(preds, tuple):\n",
        "        preds = preds[0]\n",
        "\n",
        "    decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)\n",
        "\n",
        "    # replace labels with -100 to padding\n",
        "    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n",
        "    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n",
        "\n",
        "    decoded_preds = [pred.strip() for pred in decoded_preds]\n",
        "    decoded_labels = [[label.strip()] for label in decoded_labels]\n",
        "\n",
        "    result = metric.compute(predictions=decoded_preds, references=decoded_labels)\n",
        "    return {\"bleu\": result[\"score\"]}"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's define training and validation datasets. The original dataset is quite big and training and evaluation will take quite a long time so you can run training on limited subset of the original data."
      ],
      "metadata": {
        "id": "JfaLzSMx8WN9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "train_dataset = tokenized_datasets['train']"
      ],
      "metadata": {
        "id": "zfn3fLPh8gWE"
      },
      "execution_count": 100,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "val_dataset = tokenized_datasets['validation']"
      ],
      "metadata": {
        "id": "JEyNzFOM8lV9"
      },
      "execution_count": 101,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now we can create trainer passing the model, training arguments, training dataset, validation dataset, data collator, tokenizer and compute metrics function."
      ],
      "metadata": {
        "id": "ful7Cdff22SH"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 102,
      "metadata": {
        "id": "v5-FjdTn2qbL",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1b3b1b21-a294-4c3e-8ca9-57eac0f61982"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py:432: FutureWarning: Passing the following arguments to `Accelerator` is deprecated and will be removed in version 1.0 of Accelerate: dict_keys(['dispatch_batches', 'split_batches', 'even_batches', 'use_seedable_sampler']). Please pass an `accelerate.DataLoaderConfiguration` instead: \n",
            "dataloader_config = DataLoaderConfiguration(dispatch_batches=None, split_batches=False, even_batches=True, use_seedable_sampler=True)\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "from transformers import Seq2SeqTrainer\n",
        "\n",
        "trainer = Seq2SeqTrainer(\n",
        "    model,\n",
        "    training_args,\n",
        "    train_dataset=train_dataset,\n",
        "    eval_dataset=val_dataset,\n",
        "    data_collator=data_collator,\n",
        "    tokenizer=tokenizer,\n",
        "    compute_metrics=compute_metrics,\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Task 5.**\n",
        "\n",
        "*2 points*\n",
        "\n",
        "Run evaluation of pretrained model using `evaluate` method of trainer to evaluate the model's performance before fine-tuning.\n",
        "\n",
        "Then run training with `train` method and evaluate fine-tuned model.\n",
        "\n",
        "Try to take a not too small sample of data for fine tuning. Training on around 10k examples could be ok."
      ],
      "metadata": {
        "id": "5i_KSgya9Oa_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "trainer.evaluate"
      ],
      "metadata": {
        "id": "HZLtbqy79uWh",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "89f1a500-c100-431b-99f4-efc7d5ffcacc"
      },
      "execution_count": 103,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<bound method Seq2SeqTrainer.evaluate of <transformers.trainer_seq2seq.Seq2SeqTrainer object at 0x7cf2e2458700>>"
            ],
            "text/html": [
              "<div style=\"max-width:800px; border: 1px solid var(--colab-border-color);\"><style>\n",
              "      pre.function-repr-contents {\n",
              "        overflow-x: auto;\n",
              "        padding: 8px 12px;\n",
              "        max-height: 500px;\n",
              "      }\n",
              "\n",
              "      pre.function-repr-contents.function-repr-contents-collapsed {\n",
              "        cursor: pointer;\n",
              "        max-height: 100px;\n",
              "      }\n",
              "    </style>\n",
              "    <pre style=\"white-space: initial; background:\n",
              "         var(--colab-secondary-surface-color); padding: 8px 12px;\n",
              "         border-bottom: 1px solid var(--colab-border-color);\"><b>transformers.trainer_seq2seq.Seq2SeqTrainer.evaluate</b><br/>def evaluate(eval_dataset: Optional[Dataset]=None, ignore_keys: Optional[List[str]]=None, metric_key_prefix: str=&#x27;eval&#x27;, **gen_kwargs) -&gt; Dict[str, float]</pre><pre class=\"function-repr-contents function-repr-contents-collapsed\" style=\"\"><a class=\"filepath\" style=\"display:none\" href=\"#\">/usr/local/lib/python3.10/dist-packages/transformers/trainer_seq2seq.py</a>Run evaluation and returns metrics.\n",
              "\n",
              "The calling script will be responsible for providing a method to compute metrics, as they are task-dependent\n",
              "(pass it to the init `compute_metrics` argument).\n",
              "\n",
              "You can also subclass and override this method to inject custom behavior.\n",
              "\n",
              "Args:\n",
              "    eval_dataset (`Dataset`, *optional*):\n",
              "        Pass a dataset if you wish to override `self.eval_dataset`. If it is an [`~datasets.Dataset`], columns\n",
              "        not accepted by the `model.forward()` method are automatically removed. It must implement the `__len__`\n",
              "        method.\n",
              "    ignore_keys (`List[str]`, *optional*):\n",
              "        A list of keys in the output of your model (if it is a dictionary) that should be ignored when\n",
              "        gathering predictions.\n",
              "    metric_key_prefix (`str`, *optional*, defaults to `&quot;eval&quot;`):\n",
              "        An optional prefix to be used as the metrics key prefix. For example the metrics &quot;bleu&quot; will be named\n",
              "        &quot;eval_bleu&quot; if the prefix is `&quot;eval&quot;` (default)\n",
              "    max_length (`int`, *optional*):\n",
              "        The maximum target length to use when predicting with the generate method.\n",
              "    num_beams (`int`, *optional*):\n",
              "        Number of beams for beam search that will be used when predicting with the generate method. 1 means no\n",
              "        beam search.\n",
              "    gen_kwargs:\n",
              "        Additional `generate` specific kwargs.\n",
              "\n",
              "Returns:\n",
              "    A dictionary containing the evaluation loss and the potential metrics computed from the predictions. The\n",
              "    dictionary also contains the epoch number which comes from the training state.</pre>\n",
              "      <script>\n",
              "      if (google.colab.kernel.accessAllowed && google.colab.files && google.colab.files.view) {\n",
              "        for (const element of document.querySelectorAll('.filepath')) {\n",
              "          element.style.display = 'block'\n",
              "          element.onclick = (event) => {\n",
              "            event.preventDefault();\n",
              "            event.stopPropagation();\n",
              "            google.colab.files.view(element.textContent, 112);\n",
              "          };\n",
              "        }\n",
              "      }\n",
              "      for (const element of document.querySelectorAll('.function-repr-contents')) {\n",
              "        element.onclick = (event) => {\n",
              "          event.preventDefault();\n",
              "          event.stopPropagation();\n",
              "          element.classList.toggle('function-repr-contents-collapsed');\n",
              "        };\n",
              "      }\n",
              "      </script>\n",
              "      </div>"
            ]
          },
          "metadata": {},
          "execution_count": 103
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from torch.utils.data import random_split\n",
        "\n",
        "def load_small_sample(dataset, sample_size=10000):\n",
        "    return random_split(dataset, [sample_size, len(dataset) - sample_size])[0]\n",
        "\n",
        "small_train_dataset = load_small_sample(train_dataset, sample_size=10000)"
      ],
      "metadata": {
        "id": "0b9q4iUloKIX"
      },
      "execution_count": 105,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "trainer.train_dataset=small_train_dataset\n",
        "trainer.train()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 287
        },
        "id": "fIJSzSeApF3Q",
        "outputId": "66726586-2454-4ca3-ed0d-6c5b1022cc58"
      },
      "execution_count": 108,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='939' max='939' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [939/939 02:08, Epoch 3/3]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>500</td>\n",
              "      <td>1.255700</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some non-default generation parameters are set in the model config. These should go into a GenerationConfig file (https://huggingface.co/docs/transformers/generation_strategies#save-a-custom-decoding-strategy-with-your-model) instead. This warning will be raised to an exception in v4.41.\n",
            "Non-default generation parameters: {'max_length': 512, 'num_beams': 4, 'bad_words_ids': [[59513]], 'forced_eos_token_id': 0}\n",
            "Some non-default generation parameters are set in the model config. These should go into a GenerationConfig file (https://huggingface.co/docs/transformers/generation_strategies#save-a-custom-decoding-strategy-with-your-model) instead. This warning will be raised to an exception in v4.41.\n",
            "Non-default generation parameters: {'max_length': 512, 'num_beams': 4, 'bad_words_ids': [[59513]], 'forced_eos_token_id': 0}\n",
            "Some non-default generation parameters are set in the model config. These should go into a GenerationConfig file (https://huggingface.co/docs/transformers/generation_strategies#save-a-custom-decoding-strategy-with-your-model) instead. This warning will be raised to an exception in v4.41.\n",
            "Non-default generation parameters: {'max_length': 512, 'num_beams': 4, 'bad_words_ids': [[59513]], 'forced_eos_token_id': 0}\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=939, training_loss=1.172622209412856, metrics={'train_runtime': 131.5805, 'train_samples_per_second': 227.997, 'train_steps_per_second': 7.136, 'total_flos': 253064006074368.0, 'train_loss': 1.172622209412856, 'epoch': 3.0})"
            ]
          },
          "metadata": {},
          "execution_count": 108
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "fine_tuned_evaluation_results = trainer.evaluate()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "id": "PcDANTcypUlC",
        "outputId": "70cb172d-c484-4365-ea38-5148589daf2d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='1920' max='1920' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [1920/1920 53:06]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Check the inference of the model on some examples using pipeline and analyze if the results are different after fine-tuning."
      ],
      "metadata": {
        "id": "xDtFHy5d9u4B"
      }
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "djqlPFIXHagn"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}