Texta_ai_Amazon_gpt_neo.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "Texta.ai: Amazon gpt-neo.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "38597bfea1d84b21ba5a73329b63b72f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_72bd3400e5b14f60a6655bf4cb5e8a07",
              "IPY_MODEL_e716773fc2134239a3388235f5fee174",
              "IPY_MODEL_ba21191c18854f0c94d423a0c8de5f88"
            ],
            "layout": "IPY_MODEL_21789c90fcc54372972252747f93ef28"
          }
        },
        "72bd3400e5b14f60a6655bf4cb5e8a07": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d4b1a74f2e8f4233a71d95ce6d8c76c8",
            "placeholder": "​",
            "style": "IPY_MODEL_e09f73550451466e98483d605ac77c2b",
            "value": ""
          }
        },
        "e716773fc2134239a3388235f5fee174": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dde7c8e61f4c46e592e6f642c61ae423",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_07c60754914e4add9f1d279c1b828c84",
            "value": 1
          }
        },
        "ba21191c18854f0c94d423a0c8de5f88": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d7dfefae9e574736addfd3e522228820",
            "placeholder": "​",
            "style": "IPY_MODEL_eb4fbf7c038c499d9f04db97a9fe0fdb",
            "value": " 0/? [00:00&lt;?, ? tables/s]"
          }
        },
        "21789c90fcc54372972252747f93ef28": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d4b1a74f2e8f4233a71d95ce6d8c76c8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e09f73550451466e98483d605ac77c2b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "dde7c8e61f4c46e592e6f642c61ae423": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "20px"
          }
        },
        "07c60754914e4add9f1d279c1b828c84": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d7dfefae9e574736addfd3e522228820": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "eb4fbf7c038c499d9f04db97a9fe0fdb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4eb920c6f76e4e00b0658bf9a8c6d9d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c674d5041ce94059b550c9c2d1f08e77",
              "IPY_MODEL_0d0b5a145b0647e6ab3ff3eb547f76d4",
              "IPY_MODEL_1f31583f277444d1b7b246dbf0981140"
            ],
            "layout": "IPY_MODEL_bdd5c2b38eea4eb2aab4b58fe12cb185"
          }
        },
        "c674d5041ce94059b550c9c2d1f08e77": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_73d304e0f05244dda260da54ea7b4c17",
            "placeholder": "​",
            "style": "IPY_MODEL_8effad11396441609d3abc7663905303",
            "value": "100%"
          }
        },
        "0d0b5a145b0647e6ab3ff3eb547f76d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_83b8763028054626b747c9f91fca5cd5",
            "max": 25,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_16a68ebc04da44f5a01c7691923b7e4a",
            "value": 25
          }
        },
        "1f31583f277444d1b7b246dbf0981140": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8d5443cf35484ec3bfb47223f1917f13",
            "placeholder": "​",
            "style": "IPY_MODEL_3bc58213a54e42ecb889a56e87d1f5ad",
            "value": " 25/25 [00:01&lt;00:00, 22.07ba/s]"
          }
        },
        "bdd5c2b38eea4eb2aab4b58fe12cb185": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "73d304e0f05244dda260da54ea7b4c17": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8effad11396441609d3abc7663905303": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "83b8763028054626b747c9f91fca5cd5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "16a68ebc04da44f5a01c7691923b7e4a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8d5443cf35484ec3bfb47223f1917f13": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3bc58213a54e42ecb889a56e87d1f5ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "15c187346989433f829f037f1f12b5d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_24e014a0c8d547aa989c814ecabbb19f",
              "IPY_MODEL_7ce6445125c84e85b0b6115af31e858c",
              "IPY_MODEL_66d35120aaa94ffc8ae055c6c031f4ac"
            ],
            "layout": "IPY_MODEL_766ffb48809941ccb1bae3adc38331ac"
          }
        },
        "24e014a0c8d547aa989c814ecabbb19f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4fbb7a8b480442468973e8e4ac00b0b5",
            "placeholder": "​",
            "style": "IPY_MODEL_f94b8edd23b24dc7ab0e804709fd1419",
            "value": "100%"
          }
        },
        "7ce6445125c84e85b0b6115af31e858c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_77847fe8fbed447cb36c65621cfa91a4",
            "max": 25,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0dbc40ec55454b7b983e7965a2b1767a",
            "value": 25
          }
        },
        "66d35120aaa94ffc8ae055c6c031f4ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8842e99f57d3474cb33ad72e8a5e0814",
            "placeholder": "​",
            "style": "IPY_MODEL_9bb1d7a91f2f49b5ada2adcdaec154d6",
            "value": " 25/25 [00:04&lt;00:00,  5.95ba/s]"
          }
        },
        "766ffb48809941ccb1bae3adc38331ac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4fbb7a8b480442468973e8e4ac00b0b5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f94b8edd23b24dc7ab0e804709fd1419": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "77847fe8fbed447cb36c65621cfa91a4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0dbc40ec55454b7b983e7965a2b1767a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8842e99f57d3474cb33ad72e8a5e0814": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9bb1d7a91f2f49b5ada2adcdaec154d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/nomomon/Texta.ai-Amazon-gpt-neo/blob/main/Texta_ai_Amazon_gpt_neo.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tvqNK8Rea-Dp"
      },
      "source": [
        "туториал как деплойнуть в самом низу"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p9WyC2cVZyRl"
      },
      "source": [
        "# Prepare Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yQweK8qXaYxf"
      },
      "source": [
        "import urllib.request\n",
        "\n",
        "def download_file(category):\n",
        "    metadata_url = \"http://deepyeti.ucsd.edu/jianmo/amazon/metaFiles/meta_\"+ category +\".json.gz\"\n",
        "    urllib.request.urlretrieve(metadata_url, './data.json.gz')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1dlPSXlbeXM-"
      },
      "source": [
        "import gzip\n",
        "import json\n",
        "\n",
        "def read_file():\n",
        "    counter = 0\n",
        "    data = []\n",
        "    with gzip.open('data.json.gz') as f:\n",
        "        for l in f:\n",
        "            if counter < 1000:\n",
        "                data.append(json.loads(l.strip()))\n",
        "                counter+=1\n",
        "        \n",
        "    return data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZrjY5oHoXWcg"
      },
      "source": [
        "download_file(\"Home_and_Kitchen\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rFy0FSKjXb47"
      },
      "source": [
        "a = read_file()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8Bs3ff4mXidu"
      },
      "source": [
        "a[0]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3ZSUmXllbzuv"
      },
      "source": [
        "def clean(item):\n",
        "    if type(item[\"description\"]).__name__ == \"str\":\n",
        "        item[\"description\"] = [item[\"description\"]]\n",
        "    if type(item[\"category\"]).__name__ == \"str\":\n",
        "        item[\"category\"] = [item[\"category\"]]\n",
        "    if type(item[\"feature\"]).__name__ == \"str\":\n",
        "        item[\"feature\"] = [item[\"feature\"]]\n",
        "    return item"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-wyjK14Hc9le"
      },
      "source": [
        "import re\n",
        "\n",
        "def cleanhtml(raw_html):\n",
        "  cleanr = re.compile('<.*?>')\n",
        "  cleantext = re.sub(cleanr, '', raw_html)\n",
        "  return cleantext"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fiYJcbEme2L-"
      },
      "source": [
        "def convert_to_prompt(item):\n",
        "    title = item[\"title\"]\n",
        "    categories = item[\"main_cat\"] + \", \" +\", \".join(item[\"category\"])\n",
        "    short = f\"\"\"{item[\"brand\"]}, {item[\"tech1\"]}, {item[\"tech2\"]}\"\"\"\n",
        "    feature = \"\\n- \"+\"\\n- \".join(item[\"feature\"])\n",
        "    description = \". \".join(item[\"description\"])\n",
        "\n",
        "    short = cleanhtml(short)\n",
        "    description = cleanhtml(description)\n",
        "    categories = cleanhtml(categories)\n",
        "    feature = cleanhtml(feature)\n",
        "\n",
        "\n",
        "    if len(short) > 10 and len(categories) > 5 and len(title) > 5 and len(description) > 10 and len(feature) > 10:\n",
        "        prompt = f'''\n",
        "<|endoftext|>Categories: {categories}\n",
        "Short description: {short}\n",
        "Title: {title}\n",
        "Description: {description}\n",
        "Features: {feature}\n",
        "<|endoftext|>    \n",
        "'''\n",
        "    else:\n",
        "        prompt = \"\"\n",
        "\n",
        "    return prompt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6zxchg8wzYsN"
      },
      "source": [
        "## Put to file"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JNySGkM1zbeP"
      },
      "source": [
        "def put_to_file(category, text):\n",
        "    f = open(f\"datasets/{category}.txt\", \"a+\")\n",
        "    f.write(text)\n",
        "    f.close()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R2ihHbEQ0CA9"
      },
      "source": [
        "## Merge everything"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ekWejt8C2uBT"
      },
      "source": [
        "categories = [\n",
        "    \"All Beauty\",\n",
        "    \"Appliances\",\n",
        "    \"Arts Crafts and Sewing\",\n",
        "    \"Automotive\",\n",
        "    # \"Books\",\n",
        "    \"CDs and Vinyl\",\n",
        "    \"Cell Phones and Accessories\",\n",
        "    # \"Digital Music\",\n",
        "    \"Electronics\",\n",
        "    # \"Gift Cards\",\n",
        "    \"Grocery and Gourmet Food\",\n",
        "    \"Home and Kitchen\",\n",
        "    \"Industrial and Scientific\",\n",
        "    \"Kindle Store\",\n",
        "    \"Luxury Beauty\",\n",
        "    # \"Magazine Subscriptions\",\n",
        "    # \"Movies and TV\",\n",
        "    \"Musical Instruments\",\n",
        "    \"Office Products\",\n",
        "    \"Patio Lawn and Garden\",\n",
        "    \"Pet Supplies\",\n",
        "    \"Prime Pantry\",\n",
        "    # \"Software\",\n",
        "    \"Sports and Outdoors\",\n",
        "    \"Tools and Home Improvement\",\n",
        "    \"Toys and Games\",\n",
        "    \"Video Games\"\n",
        "]\n",
        "\n",
        "done = []"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RLshfP7wO5rq"
      },
      "source": [
        "import os, shutil\n",
        "folder = 'datasets'\n",
        "for filename in os.listdir(folder):\n",
        "    file_path = os.path.join(folder, filename)\n",
        "    try:\n",
        "        if os.path.isfile(file_path) or os.path.islink(file_path):\n",
        "            os.unlink(file_path)\n",
        "        elif os.path.isdir(file_path):\n",
        "            shutil.rmtree(file_path)\n",
        "    except Exception as e:\n",
        "        print('Failed to delete %s. Reason: %s' % (file_path, e))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2C8pv4Uz0Abl"
      },
      "source": [
        "def doAll():\n",
        "    for category in categories:\n",
        "        if category not in done:    \n",
        "            print(f\"Doing {category} ...\")\n",
        "\n",
        "            subCat = category.replace(\" \", \"_\")\n",
        "            download_file(subCat)\n",
        "            print(\"     Downloaded\")\n",
        "            for item in read_file():\n",
        "                put_to_file(subCat, convert_to_prompt(clean(item)))\n",
        "            print(\"     Placed in a file\")\n",
        "\n",
        "            done.append(category)\n",
        "\n",
        "doAll()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z7YCYSRZT6XY"
      },
      "source": [
        "## Save the dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DFMGG5rrURNQ"
      },
      "source": [
        "import os\n",
        "import zipfile\n",
        "    \n",
        "def zipdir(path, ziph):\n",
        "    # ziph is zipfile handle\n",
        "    for root, dirs, files in os.walk(path):\n",
        "        for file in files:\n",
        "            ziph.write(os.path.join(root, file), \n",
        "                       os.path.relpath(os.path.join(root, file), \n",
        "                                       os.path.join(path, '..')))\n",
        "      \n",
        "zipf = zipfile.ZipFile('dataset.zip', 'w', zipfile.ZIP_DEFLATED)\n",
        "zipdir('datasets/', zipf)\n",
        "zipf.close()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "te-iKjHQT61k",
        "outputId": "db967495-f4c9-4ac7-db58-46d2187a68f0"
      },
      "source": [
        "from google.colab import files\n",
        "files.download('dataset.zip') "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "download(\"download_ad29e444-b348-4b22-9b29-b63e145be9cc\", \"dataset.zip\", 57651094)"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CFd5Npj8Dzlm"
      },
      "source": [
        "# GPT-NEO"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QBvUIXItorY6"
      },
      "source": [
        "pip install happytransformer"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LjvhF70wowBF"
      },
      "source": [
        "from happytransformer import HappyGeneration"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zmbkr9fEZ3Xa"
      },
      "source": [
        "# Train GPT-Neo"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jax6bg9KqQcr"
      },
      "source": [
        "# happy_gen = HappyGeneration(\"GPT-NEO\", \"EleutherAI/gpt-neo-1.3B\")\n",
        "happy_gen = HappyGeneration(\"GPT-NEO\", \"EleutherAI/gpt-neo-125M\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "h_wFq3kS09t0"
      },
      "source": [
        "Add a text file called \"train.txt\" to the environment. Also, downgrade to the model to \"EleutherAI/gpt-neo-125M\" model so that the environment has enough memory    "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wYZ64OTSFQKY"
      },
      "source": [
        "from happytransformer import GENTrainArgs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/",
          "height": 479,
          "referenced_widgets": [
            "38597bfea1d84b21ba5a73329b63b72f",
            "72bd3400e5b14f60a6655bf4cb5e8a07",
            "e716773fc2134239a3388235f5fee174",
            "ba21191c18854f0c94d423a0c8de5f88",
            "21789c90fcc54372972252747f93ef28",
            "d4b1a74f2e8f4233a71d95ce6d8c76c8",
            "e09f73550451466e98483d605ac77c2b",
            "dde7c8e61f4c46e592e6f642c61ae423",
            "07c60754914e4add9f1d279c1b828c84",
            "d7dfefae9e574736addfd3e522228820",
            "eb4fbf7c038c499d9f04db97a9fe0fdb",
            "4eb920c6f76e4e00b0658bf9a8c6d9d5",
            "c674d5041ce94059b550c9c2d1f08e77",
            "0d0b5a145b0647e6ab3ff3eb547f76d4",
            "1f31583f277444d1b7b246dbf0981140",
            "bdd5c2b38eea4eb2aab4b58fe12cb185",
            "73d304e0f05244dda260da54ea7b4c17",
            "8effad11396441609d3abc7663905303",
            "83b8763028054626b747c9f91fca5cd5",
            "16a68ebc04da44f5a01c7691923b7e4a",
            "8d5443cf35484ec3bfb47223f1917f13",
            "3bc58213a54e42ecb889a56e87d1f5ad",
            "15c187346989433f829f037f1f12b5d2",
            "24e014a0c8d547aa989c814ecabbb19f",
            "7ce6445125c84e85b0b6115af31e858c",
            "66d35120aaa94ffc8ae055c6c031f4ac",
            "766ffb48809941ccb1bae3adc38331ac",
            "4fbb7a8b480442468973e8e4ac00b0b5",
            "f94b8edd23b24dc7ab0e804709fd1419",
            "77847fe8fbed447cb36c65621cfa91a4",
            "0dbc40ec55454b7b983e7965a2b1767a",
            "8842e99f57d3474cb33ad72e8a5e0814",
            "9bb1d7a91f2f49b5ada2adcdaec154d6"
          ]
        },
        "id": "rbg6hzHMDFAg",
        "outputId": "2a46e42f-d8ce-498e-aac6-345152fb525a"
      },
      "source": [
        "train_args = GENTrainArgs(learning_rate = 3e-6, num_train_epochs = 1)\n",
        "happy_gen.train(\"merge_from_ofoct (1).txt\", args = train_args)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "08/15/2021 09:10:31 - INFO - happytransformer.happy_transformer -   Preprocessing dataset...\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Downloading and preparing dataset text/default (download: Unknown size, generated: Unknown size, post-processed: Unknown size, total: Unknown size) to /root/.cache/huggingface/datasets/text/default-1e781b6cd0fac616/0.0.0/e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "38597bfea1d84b21ba5a73329b63b72f",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "0 tables [00:00, ? tables/s]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Dataset text downloaded and prepared to /root/.cache/huggingface/datasets/text/default-1e781b6cd0fac616/0.0.0/e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5. Subsequent calls will reuse this data.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4eb920c6f76e4e00b0658bf9a8c6d9d5",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/25 [00:00<?, ?ba/s]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "15c187346989433f829f037f1f12b5d2",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/25 [00:00<?, ?ba/s]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "08/15/2021 09:10:37 - INFO - happytransformer.happy_transformer -   Training...\n",
            "PyTorch: setting up devices\n",
            "***** Running training *****\n",
            "  Num examples = 708\n",
            "  Num Epochs = 1\n",
            "  Instantaneous batch size per device = 1\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 1\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 708\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='708' max='708' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [708/708 02:28, Epoch 1/1]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>500</td>\n",
              "      <td>3.028600</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving model checkpoint to /tmp/tmpxztc1ahz/checkpoint-500\n",
            "Configuration saved in /tmp/tmpxztc1ahz/checkpoint-500/config.json\n",
            "Model weights saved in /tmp/tmpxztc1ahz/checkpoint-500/pytorch_model.bin\n",
            "tokenizer config file saved in /tmp/tmpxztc1ahz/checkpoint-500/tokenizer_config.json\n",
            "Special tokens file saved in /tmp/tmpxztc1ahz/checkpoint-500/special_tokens_map.json\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3TQruZ-y6URo"
      },
      "source": [
        "# happy_gen.save(\"/content/drive/MyDrive/GPT-Neo_Amazon/3/\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YVqMI5s2z39r"
      },
      "source": [
        "# Test Model\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9ho167YF6a8W"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lghVoCEoVkPx"
      },
      "source": [
        "from happytransformer import HappyGeneration\n",
        "\n",
        "happy_gen = HappyGeneration(load_path=\"/content/drive/MyDrive/GPT-Neo_Amazon/\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a-rslcEDs2yX"
      },
      "source": [
        "from happytransformer import GENSettings"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yVWSPOZU4jvk"
      },
      "source": [
        "#@title Settings\n",
        "\n",
        "# https://happytransformer.com/text-generation/settings/\n",
        "\n",
        "min_length =  10#@param {type:\"number\"}\n",
        "max_length = 20 #@param {type:\"number\"}\n",
        "do_sample = True #@param {type:\"boolean\"}\n",
        "early_stopping = False #@param {type:\"boolean\"}\n",
        "num_beams = 1 #@param {type:\"number\"}\n",
        "temperature = 0.51 #@param {type:\"slider\", min:0, max:1, step:0.1}\n",
        "top_k = 50 #@param {type:\"slider\", min:1, max:50, step:1}\n",
        "top_p = 0.9 #@param {type:\"slider\", min:0, max:1, step:0.1}\n",
        "no_repeat_ngram_size = 2 #@param {type:\"slider\", min:0, max:10, step:1}\n",
        "\n",
        "gen_args = GENSettings(min_length, max_length, do_sample, early_stopping, num_beams, temperature, top_k, no_repeat_ngram_size, top_p)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-dccnwV5ux5l",
        "outputId": "c5f2a5cf-6eb6-43a7-ae99-3c259f07a557"
      },
      "source": [
        "#@title Generate **Title** for Product\n",
        "\n",
        "category = \"Sports and Outdoors\"#@param [\"All Beauty\", \"Appliances\", \"Arts Crafts and Sewing\", \"Automotive\", \"CDs and Vinyl\", \"Cell Phones and Accessories\", \"Electronics\", \"Grocery and Gourmet Food\", \"Home and Kitchen\", \"Industrial and Scientific\", \"Kindle Store\", \"Luxury Beauty\", \"Musical Instruments\", \"Office Products\", \"Patio Lawn and Garden\", \"Pet Supplies\", \"Prime Pantry\", \"Sports and Outdoors\", \"Tools and Home Improvement\", \"Toys and Games\", \"Video Games\"]\n",
        "short = \"Fit and Fresh, Cool Coolers Reusable Ice Packs, Set of 4, Clear, Penguins\" #@param {type:\"string\"}\n",
        "n_examples = 5 #@param {type:\"number\"}\n",
        "\n",
        "text = f\"\"\"Category: {category}\n",
        "Short description:{short}\n",
        "Title: \"\"\"\n",
        "\n",
        "output = \"\\n\\n\\n\"\n",
        "for i in range(n_examples):\n",
        "    result = happy_gen.generate_text(text, args=gen_args)\n",
        "\n",
        "    output += '\\n\\n>>> ' + result.text.split(\"Features\")[0].split(\"Description\")[0]\n",
        "\n",
        "print(output)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            ">>>   Ice Pack with Penguins and Penguins\n",
            "\n",
            ">>>   Ice Pack of 5 - 4 Pack - Clear Penguins - Penguins Icepacks\n",
            "\n",
            ">>>   Ice Pack - 5 Pack of Icepack Icepacks, Ice pack with Ice packs\n",
            "\n",
            ">>>   Ice Pack, Icepack Icepacks, 1 Pack\n",
            "\n",
            ">>>  Ice Packets, Ice pack, reusable ice pack with a pocket, 2 Pack\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "poMk7tQs-SMp"
      },
      "source": [
        "#@title Settings\n",
        "\n",
        "# https://happytransformer.com/text-generation/settings/\n",
        "\n",
        "min_length =  20#@param {type:\"number\"}\n",
        "max_length = 50 #@param {type:\"number\"}\n",
        "do_sample = True #@param {type:\"boolean\"}\n",
        "early_stopping = False #@param {type:\"boolean\"}\n",
        "num_beams = 1 #@param {type:\"number\"}\n",
        "temperature = 0.7 #@param {type:\"slider\", min:0, max:1, step:0.1}\n",
        "top_k = 50 #@param {type:\"slider\", min:1, max:50, step:1}\n",
        "top_p = 1 #@param {type:\"slider\", min:0, max:1, step:0.1}\n",
        "no_repeat_ngram_size = 2 #@param {type:\"slider\", min:0, max:10, step:1}\n",
        "\n",
        "gen_args = GENSettings(min_length, max_length, do_sample, early_stopping, num_beams, temperature, top_k, no_repeat_ngram_size, top_p)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3R-uFaHOwSTO",
        "outputId": "7fc33c82-7ab7-4afe-b088-520bd67b9ec5"
      },
      "source": [
        "#@title Generate **Description** for Product\n",
        "\n",
        "category = \"Sports and Outdoors\"#@param [\"All Beauty\", \"Appliances\", \"Arts Crafts and Sewing\", \"Automotive\", \"CDs and Vinyl\", \"Cell Phones and Accessories\", \"Electronics\", \"Grocery and Gourmet Food\", \"Home and Kitchen\", \"Industrial and Scientific\", \"Kindle Store\", \"Luxury Beauty\", \"Musical Instruments\", \"Office Products\", \"Patio Lawn and Garden\", \"Pet Supplies\", \"Prime Pantry\", \"Sports and Outdoors\", \"Tools and Home Improvement\", \"Toys and Games\", \"Video Games\"]\n",
        "short = \"5.25 x 3 x 5.5 inches, 1 pounds, Fit and Fresh, 336KFF, #6 in Sports & Outdoors (See Top 100 in Sports & Outdoors) #1 in Camping Cooler Accessories, , SLIM ICE PACKS, COMPACT DESIGN, LONG-LASTING, HEALTHY AND SAFE\" #@param {type:\"string\"}\n",
        "title = \"Fit & Fresh Cool Coolers Reusable Ice Packs, Set of 4, Clear, Package may vary\" #@param {type:\"string\"}\n",
        "n_examples = 5 #@param {type:\"number\"}\n",
        "\n",
        "text = f\"\"\"Category: {category}\n",
        "Short description: {short}\n",
        "Title: {title}\n",
        "Description: \"\"\"\n",
        "\n",
        "output = \"\\n\\n\\n\"\n",
        "for i in range(n_examples):\n",
        "    result = happy_gen.generate_text(text, args=gen_args)\n",
        "\n",
        "    output += '\\n\\n>>> ' + result.text.split(\"Features\")[0]\n",
        "\n",
        "print(output)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            ">>>  Fit-Fresh Cooling is a great way to stay cool in your outdoor life and use it on a variety of outdoor activities.  The frozen-cooler is made with a reusable ice pack that can be used as a cool-down item or\n",
            "\n",
            ">>>  Fit&Fresh Cooling is a lightweight, portable ice pack that fits on most patio decks. It is designed in a compact package with a clean, clean fit and fresh design.  It contains 4 stainless steel bottles, as well as a large metal\n",
            "\n",
            ">>>  -  Fit Ice Pack  2.0oz. IcePack  4.8oz Icepack  1.6oz ice pack  Package included  \n",
            "\n",
            ">>>  1. The Fit&Fresh Cooling Reusable Ice Pack  with the Fresh and Clear Package  is easy to clean, durable, and contains enough ice to last you for years  2. Each pack contains 3.6 ounces of ice,\n",
            "\n",
            ">>>  Fit&Fresh is a cool new and easy-to-use ice pack that's easy to clean, and a great ice scraper that helps you remove ice from your water bottles.  Its perfect for cleaning and having a blast with your family.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F1gS5ZtA-Q9M"
      },
      "source": [
        "#@title Settings for Features (BulletPoints)\n",
        "\n",
        "# https://happytransformer.com/text-generation/settings/\n",
        "\n",
        "min_length =  10#@param {type:\"number\"}\n",
        "max_length = 100 #@param {type:\"number\"}\n",
        "do_sample = True #@param {type:\"boolean\"}\n",
        "early_stopping = True #@param {type:\"boolean\"}\n",
        "num_beams = 1 #@param {type:\"number\"}\n",
        "temperature = 0.6 #@param {type:\"slider\", min:0, max:1, step:0.1}\n",
        "top_k = 50 #@param {type:\"slider\", min:1, max:50, step:1}\n",
        "top_p = 0.8 #@param {type:\"slider\", min:0, max:1, step:0.1}\n",
        "no_repeat_ngram_size = 1 #@param {type:\"slider\", min:0, max:10, step:1}\n",
        "\n",
        "gen_args = GENSettings(min_length, max_length, do_sample, early_stopping, num_beams, temperature, top_k, no_repeat_ngram_size, top_p)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aX0DnZcvwjHE",
        "cellView": "form",
        "outputId": "077a9998-9458-49c9-8d0a-ea32febf17d8"
      },
      "source": [
        "#@title Generate **Features** for Product\n",
        "\n",
        "category = \"Sports and Outdoors\"#@param [\"All Beauty\", \"Appliances\", \"Arts Crafts and Sewing\", \"Automotive\", \"CDs and Vinyl\", \"Cell Phones and Accessories\", \"Electronics\", \"Grocery and Gourmet Food\", \"Home and Kitchen\", \"Industrial and Scientific\", \"Kindle Store\", \"Luxury Beauty\", \"Musical Instruments\", \"Office Products\", \"Patio Lawn and Garden\", \"Pet Supplies\", \"Prime Pantry\", \"Sports and Outdoors\", \"Tools and Home Improvement\", \"Toys and Games\", \"Video Games\"]\n",
        "short = \"5.25 x 3 x 5.5 inches, 1 pounds, Fit and Fresh, 336KFF, #6 in Sports & Outdoors (See Top 100 in Sports & Outdoors) #1 in Camping Cooler Accessories, , SLIM ICE PACKS, COMPACT DESIGN, LONG-LASTING, HEALTHY AND SAFE\" #@param {type:\"string\"}\n",
        "title = \"Fit & Fresh Cool Coolers Reusable Ice Packs, Set of 4, Clear, Package may vary\" #@param {type:\"string\"}\n",
        "description = \"Fit Fresh and freeze, and take up to 70 pounds of ice, so you can stay cool. Use ice packs to cool other ice to the point that it is warmer than ice. The ice pack is manufactured with ice and foam insulation, but it does not contain ice cubes. It contains snow cubes, ice foam, snow cushions and ice gel. Do not use ice-gaugeippers to get ice from the icepack. Ice gel and snow foam are designed to resist any contact with water or ice in the area.\" #@param {type:\"string\"}\n",
        "n_examples =  5#@param {type:\"number\"}\n",
        "\n",
        "text = f\"\"\"\n",
        "Category: {category}\n",
        "Short description: f{short}\n",
        "Title: {title}\n",
        "Description: {description}\n",
        "Features:\n",
        "- \"\"\"\n",
        "\n",
        "output = \"\\n\\n\"\n",
        "for i in range(n_examples):\n",
        "    result = happy_gen.generate_text(text, args=gen_args)\n",
        "\n",
        "    output += '\\n>>>'\n",
        "    output += '\\n-' + \"\\n- \".join(result.text.split(\"- \"))\n",
        "print(output)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "\n",
            ">>>\n",
            "- \n",
            "- High quality foams for better protection against freezing weather conditions\n",
            ">>>\n",
            "- High quality foams for a great fit \n",
            "- Foam protects against bumps by preventing them getting into your boat's bottom\n",
            ">>>\n",
            "- Easy installation \n",
            "- Easily install a fit fresh winter cold hydration kit on your freezer door\n",
            ">>>\n",
            "- Perfect for outdoor activities like camping \n",
            "- fits into most winter sports equipment\n",
            ">>>\n",
            "- Perfect for outdoor camping \n",
            "- Includes all four compartments on your car windshield wiper blades; compact design provides a natural fit around most cars' heads\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ECisx_9PWlPv"
      },
      "source": [
        "# Как запускать и получать ответы от модели? (для @Rash)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "467tLPkzWseg"
      },
      "source": [
        "[Видео туториал](https://www.youtube.com/watch?v=GzHJ3NUVtV4)\n",
        "\n",
        "Все не так уж и сложно. Для начала скачаем нужную библиотеку.\n",
        "\n",
        "```\n",
        "!pip install happytransformer\n",
        "```\n",
        "Скачиваем модельку с [drive](https://drive.google.com/drive/folders/1a4SclxrGzdjrNlG4sUT3Wzpyn8qqxWLu?usp=sharing). Тут всего 4 варианта (0 тренировалась меньше всех, 3 больше).\n",
        "\n",
        "Подклучаем эту библиотеку и загружаем выбранную модельку, указывая на нее путь в `directory`.\n",
        "\n",
        "```\n",
        "from happytransformer import HappyGeneration\n",
        "\n",
        "happy_gen = HappyGeneration(load_path=\"/content/drive/MyDrive/GPT-Neo_Amazon/3/\")\n",
        "```\n",
        "\n",
        "Далее, можно задать настройки генирации. Арман в этом шарит. Но если все же есть вопросы, что это за настройки, то можно посмотреть их [тут](https://happytransformer.com/text-generation/settings/).\n",
        "```\n",
        "min_length =  10\n",
        "max_length = 100 \n",
        "do_sample = True\n",
        "early_stopping = True\n",
        "num_beams = 1 \n",
        "temperature = 0.6\n",
        "top_k = 50\n",
        "top_p = 0.8\n",
        "no_repeat_ngram_size = 1\n",
        "\n",
        "gen_args = GENSettings(min_length, max_length, do_sample, early_stopping, num_beams, temperature, top_k, no_repeat_ngram_size, top_p)\n",
        "\n",
        "```\n",
        "\n",
        "Для генерации, запускаем следующию функцию, где `text` это промпт который используем. Примеры таких можно найти в вкладке Test Model этого файла. Результат генерации находится в `result.text`.\n",
        "\n",
        "```\n",
        "result = happy_gen.generate_text(text, args=gen_args)\n",
        "\n",
        "print(result.text)\n",
        "```\n",
        "\n",
        "Вроде все. Спасибо за внимание. Было приятно с вами поработать :)"
      ]
    }
  ]
}