quest-pytorch-transformer.ipynb

{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"086421f7eec44c769f08f5f68fafafdc":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"0e7c81c0da784e04b530236bad005c33":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"18b951cd8c1447eaa1e12f972ac37d42":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"danger","description":"  9%","description_tooltip":null,"layout":"IPY_MODEL_9e743ecd26cd4877a539f9bbbd23d114","max":308,"min":0,"orientation":"horizontal","style":"IPY_MODEL_e34fafda1e234d9c981322beff1068e7","value":29}},"35645b239e914aee807cfdc234fc5b75":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_dfdee9b109bc4176a0bc3a26ae38f7a3","placeholder":"​","style":"IPY_MODEL_086421f7eec44c769f08f5f68fafafdc","value":" 29/308 [7:12:46&lt;66:18:40, 855.63s/it]"}},"554271f3ed5d48efa844608a0b72fdac":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"685ecca481e3463d83a2465f15f7b33f":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"100%","description_tooltip":null,"layout":"IPY_MODEL_80ae0d356c3e4b1bbe511c5f5a2694b5","max":4059,"min":0,"orientation":"horizontal","style":"IPY_MODEL_a34804b28be74d7c9972a0a13410ba77","value":4059}},"7f49d7d685554687bc2d131959058cb3":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"80ae0d356c3e4b1bbe511c5f5a2694b5":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"840e74b11eb44a58b1ed0433e924dc82":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"9e743ecd26cd4877a539f9bbbd23d114":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a34804b28be74d7c9972a0a13410ba77":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":"initial"}},"a61c8f56dbd74e29a155fad0d7095f79":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_18b951cd8c1447eaa1e12f972ac37d42","IPY_MODEL_35645b239e914aee807cfdc234fc5b75"],"layout":"IPY_MODEL_7f49d7d685554687bc2d131959058cb3"}},"c9ab84b07a32426282543b5ff054ddec":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_840e74b11eb44a58b1ed0433e924dc82","placeholder":"​","style":"IPY_MODEL_554271f3ed5d48efa844608a0b72fdac","value":" 4059/4059 [14:53&lt;00:00,  4.54it/s]"}},"dfdee9b109bc4176a0bc3a26ae38f7a3":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e34fafda1e234d9c981322beff1068e7":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":"initial"}},"fd692cff30e343aa8afa007a05d66acc":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_685ecca481e3463d83a2465f15f7b33f","IPY_MODEL_c9ab84b07a32426282543b5ff054ddec"],"layout":"IPY_MODEL_0e7c81c0da784e04b530236bad005c33"}}},"version_major":2,"version_minor":0}}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Pytorch BERT baseline**","metadata":{}},{"cell_type":"markdown","source":"In this version, I convert https://www.kaggle.com/akensert/bert-base-tf2-0-minimalistic into pytorch version","metadata":{}},{"cell_type":"markdown","source":"**Please upvote the kernel if you find it helpful**","metadata":{}},{"cell_type":"markdown","source":"### Install HuggingFace transformers & sacremoses dependency","metadata":{}},{"cell_type":"markdown","source":"As we are not allowed to use internet I've created required datasets and commands to setup Hugging Face Transformers setup in offline mode. You can find the required github codebases in the datasets.\n\n* sacremoses dependency - https://www.kaggle.com/axel81/sacremoses\n* transformers - https://www.kaggle.com/axel81/transformers","metadata":{}},{"cell_type":"code","source":"!pip install ../input/sacremoses/sacremoses-master/\n!pip install ../input/transformers/transformers-master/","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-04-12T19:13:56.183115Z","iopub.execute_input":"2022-04-12T19:13:56.183523Z","iopub.status.idle":"2022-04-12T19:14:10.279781Z","shell.execute_reply.started":"2022-04-12T19:13:56.183468Z","shell.execute_reply":"2022-04-12T19:14:10.278874Z"},"trusted":true},"execution_count":73,"outputs":[{"name":"stdout","text":"Processing /kaggle/input/sacremoses/sacremoses-master\nRequirement already satisfied: six in /opt/conda/lib/python3.6/site-packages (from sacremoses==0.0.35) (1.12.0)\nRequirement already satisfied: click in /opt/conda/lib/python3.6/site-packages (from sacremoses==0.0.35) (7.0)\nRequirement already satisfied: joblib in /opt/conda/lib/python3.6/site-packages (from sacremoses==0.0.35) (0.13.2)\nRequirement already satisfied: tqdm in /opt/conda/lib/python3.6/site-packages (from sacremoses==0.0.35) (4.36.1)\nBuilding wheels for collected packages: sacremoses\n  Building wheel for sacremoses (setup.py) ... \u001b[?25ldone\n\u001b[?25h  Created wheel for sacremoses: filename=sacremoses-0.0.35-cp36-none-any.whl size=882724 sha256=99ca362a450a785d5688bce99182cf4a525eef5913a5e012848b90eaa6bfaf73\n  Stored in directory: /root/.cache/pip/wheels/82/48/4b/05cb49d913a40c9d76f97931cd747d72fb17a77b0f6415cdba\nSuccessfully built sacremoses\nInstalling collected packages: sacremoses\n  Found existing installation: sacremoses 0.0.35\n    Uninstalling sacremoses-0.0.35:\n      Successfully uninstalled sacremoses-0.0.35\nSuccessfully installed sacremoses-0.0.35\nProcessing /kaggle/input/transformers/transformers-master\nRequirement already satisfied: numpy in /opt/conda/lib/python3.6/site-packages (from transformers==2.1.1) (1.16.4)\nRequirement already satisfied: boto3 in /opt/conda/lib/python3.6/site-packages (from transformers==2.1.1) (1.10.2)\nRequirement already satisfied: requests in /opt/conda/lib/python3.6/site-packages (from transformers==2.1.1) (2.22.0)\nRequirement already satisfied: tqdm in /opt/conda/lib/python3.6/site-packages (from transformers==2.1.1) (4.36.1)\nRequirement already satisfied: regex in /opt/conda/lib/python3.6/site-packages (from transformers==2.1.1) (2019.8.19)\nRequirement already satisfied: sentencepiece in /opt/conda/lib/python3.6/site-packages (from transformers==2.1.1) (0.1.83)\nRequirement already satisfied: sacremoses in /opt/conda/lib/python3.6/site-packages (from transformers==2.1.1) (0.0.35)\nRequirement already satisfied: s3transfer<0.3.0,>=0.2.0 in /opt/conda/lib/python3.6/site-packages (from boto3->transformers==2.1.1) (0.2.1)\nRequirement already satisfied: jmespath<1.0.0,>=0.7.1 in /opt/conda/lib/python3.6/site-packages (from boto3->transformers==2.1.1) (0.9.4)\nRequirement already satisfied: botocore<1.14.0,>=1.13.2 in /opt/conda/lib/python3.6/site-packages (from boto3->transformers==2.1.1) (1.13.2)\nRequirement already satisfied: chardet<3.1.0,>=3.0.2 in /opt/conda/lib/python3.6/site-packages (from requests->transformers==2.1.1) (3.0.4)\nRequirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /opt/conda/lib/python3.6/site-packages (from requests->transformers==2.1.1) (1.24.2)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.6/site-packages (from requests->transformers==2.1.1) (2019.9.11)\nRequirement already satisfied: idna<2.9,>=2.5 in /opt/conda/lib/python3.6/site-packages (from requests->transformers==2.1.1) (2.8)\nRequirement already satisfied: click in /opt/conda/lib/python3.6/site-packages (from sacremoses->transformers==2.1.1) (7.0)\nRequirement already satisfied: six in /opt/conda/lib/python3.6/site-packages (from sacremoses->transformers==2.1.1) (1.12.0)\nRequirement already satisfied: joblib in /opt/conda/lib/python3.6/site-packages (from sacremoses->transformers==2.1.1) (0.13.2)\nRequirement already satisfied: python-dateutil<3.0.0,>=2.1; python_version >= \"2.7\" in /opt/conda/lib/python3.6/site-packages (from botocore<1.14.0,>=1.13.2->boto3->transformers==2.1.1) (2.8.0)\nRequirement already satisfied: docutils<0.16,>=0.10 in /opt/conda/lib/python3.6/site-packages (from botocore<1.14.0,>=1.13.2->boto3->transformers==2.1.1) (0.15.2)\nBuilding wheels for collected packages: transformers\n  Building wheel for transformers (setup.py) ... \u001b[?25ldone\n\u001b[?25h  Created wheel for transformers: filename=transformers-2.1.1-cp36-none-any.whl size=334890 sha256=5cb3d338d68283d86434ebc4e092c7754ff21f1c41ad05ed12246de84acc2061\n  Stored in directory: /root/.cache/pip/wheels/ce/f3/1a/ee7248890cb4b8e8975988b1a67999e2d09ef54ce8ee815255\nSuccessfully built transformers\nInstalling collected packages: transformers\n  Found existing installation: transformers 2.1.1\n    Uninstalling transformers-2.1.1:\n      Successfully uninstalled transformers-2.1.1\nSuccessfully installed transformers-2.1.1\n","output_type":"stream"}]},{"cell_type":"markdown","source":"### Required Imports\n\nI've added imports that will be used in training too","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport matplotlib.pyplot as plt\nDATA_DIR = '../input/google-quest-challenge'","metadata":{"execution":{"iopub.status.busy":"2022-04-12T19:14:10.283246Z","iopub.execute_input":"2022-04-12T19:14:10.283611Z","iopub.status.idle":"2022-04-12T19:14:10.288515Z","shell.execute_reply.started":"2022-04-12T19:14:10.283537Z","shell.execute_reply":"2022-04-12T19:14:10.287653Z"},"trusted":true},"execution_count":74,"outputs":[]},{"cell_type":"code","source":"!ls ../input","metadata":{"execution":{"iopub.status.busy":"2022-04-12T19:14:10.289918Z","iopub.execute_input":"2022-04-12T19:14:10.290492Z","iopub.status.idle":"2022-04-12T19:14:10.998331Z","shell.execute_reply.started":"2022-04-12T19:14:10.290444Z","shell.execute_reply":"2022-04-12T19:14:10.997412Z"},"trusted":true},"execution_count":75,"outputs":[{"name":"stdout","text":"d  google-quest-challenge  quest-models  sacremoses  transformers\n","output_type":"stream"}]},{"cell_type":"code","source":"sub = pd.read_csv(f'{DATA_DIR}/sample_submission.csv')\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-12T19:14:11.000148Z","iopub.execute_input":"2022-04-12T19:14:11.000524Z","iopub.status.idle":"2022-04-12T19:14:11.045073Z","shell.execute_reply.started":"2022-04-12T19:14:11.000457Z","shell.execute_reply":"2022-04-12T19:14:11.044226Z"},"trusted":true},"execution_count":76,"outputs":[{"execution_count":76,"output_type":"execute_result","data":{"text/plain":"   qa_id  question_asker_intent_understanding  question_body_critical  \\\n0     39                              0.00308                 0.00308   \n1     46                              0.00448                 0.00448   \n2     70                              0.00673                 0.00673   \n3    132                              0.01401                 0.01401   \n4    200                              0.02074                 0.02074   \n\n   question_conversational  question_expect_short_answer  \\\n0                  0.00308                       0.00308   \n1                  0.00448                       0.00448   \n2                  0.00673                       0.00673   \n3                  0.01401                       0.01401   \n4                  0.02074                       0.02074   \n\n   question_fact_seeking  question_has_commonly_accepted_answer  \\\n0                0.00308                                0.00308   \n1                0.00448                                0.00448   \n2                0.00673                                0.00673   \n3                0.01401                                0.01401   \n4                0.02074                                0.02074   \n\n   question_interestingness_others  question_interestingness_self  \\\n0                          0.00308                        0.00308   \n1                          0.00448                        0.00448   \n2                          0.00673                        0.00673   \n3                          0.01401                        0.01401   \n4                          0.02074                        0.02074   \n\n   question_multi_intent  ...  question_well_written  answer_helpful  \\\n0                0.00308  ...                0.00308         0.00308   \n1                0.00448  ...                0.00448         0.00448   \n2                0.00673  ...                0.00673         0.00673   \n3                0.01401  ...                0.01401         0.01401   \n4                0.02074  ...                0.02074         0.02074   \n\n   answer_level_of_information  answer_plausible  answer_relevance  \\\n0                      0.00308           0.00308           0.00308   \n1                      0.00448           0.00448           0.00448   \n2                      0.00673           0.00673           0.00673   \n3                      0.01401           0.01401           0.01401   \n4                      0.02074           0.02074           0.02074   \n\n   answer_satisfaction  answer_type_instructions  answer_type_procedure  \\\n0              0.00308                   0.00308                0.00308   \n1              0.00448                   0.00448                0.00448   \n2              0.00673                   0.00673                0.00673   \n3              0.01401                   0.01401                0.01401   \n4              0.02074                   0.02074                0.02074   \n\n   answer_type_reason_explanation  answer_well_written  \n0                         0.00308              0.00308  \n1                         0.00448              0.00448  \n2                         0.00673              0.00673  \n3                         0.01401              0.01401  \n4                         0.02074              0.02074  \n\n[5 rows x 31 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>qa_id</th>\n      <th>question_asker_intent_understanding</th>\n      <th>question_body_critical</th>\n      <th>question_conversational</th>\n      <th>question_expect_short_answer</th>\n      <th>question_fact_seeking</th>\n      <th>question_has_commonly_accepted_answer</th>\n      <th>question_interestingness_others</th>\n      <th>question_interestingness_self</th>\n      <th>question_multi_intent</th>\n      <th>...</th>\n      <th>question_well_written</th>\n      <th>answer_helpful</th>\n      <th>answer_level_of_information</th>\n      <th>answer_plausible</th>\n      <th>answer_relevance</th>\n      <th>answer_satisfaction</th>\n      <th>answer_type_instructions</th>\n      <th>answer_type_procedure</th>\n      <th>answer_type_reason_explanation</th>\n      <th>answer_well_written</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>39</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>...</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>46</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>...</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>70</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>...</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>132</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>...</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>200</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>...</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 31 columns</p>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"target_columns = sub.columns.values[1:].tolist()\ntarget_columns","metadata":{"execution":{"iopub.status.busy":"2022-04-12T19:14:11.049527Z","iopub.execute_input":"2022-04-12T19:14:11.049787Z","iopub.status.idle":"2022-04-12T19:14:11.056648Z","shell.execute_reply.started":"2022-04-12T19:14:11.049741Z","shell.execute_reply":"2022-04-12T19:14:11.055718Z"},"trusted":true},"execution_count":77,"outputs":[{"execution_count":77,"output_type":"execute_result","data":{"text/plain":"['question_asker_intent_understanding',\n 'question_body_critical',\n 'question_conversational',\n 'question_expect_short_answer',\n 'question_fact_seeking',\n 'question_has_commonly_accepted_answer',\n 'question_interestingness_others',\n 'question_interestingness_self',\n 'question_multi_intent',\n 'question_not_really_a_question',\n 'question_opinion_seeking',\n 'question_type_choice',\n 'question_type_compare',\n 'question_type_consequence',\n 'question_type_definition',\n 'question_type_entity',\n 'question_type_instructions',\n 'question_type_procedure',\n 'question_type_reason_explanation',\n 'question_type_spelling',\n 'question_well_written',\n 'answer_helpful',\n 'answer_level_of_information',\n 'answer_plausible',\n 'answer_relevance',\n 'answer_satisfaction',\n 'answer_type_instructions',\n 'answer_type_procedure',\n 'answer_type_reason_explanation',\n 'answer_well_written']"},"metadata":{}}]},{"cell_type":"markdown","source":"### Define dataset","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(f'{DATA_DIR}/train.csv')\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-12T19:14:11.059855Z","iopub.execute_input":"2022-04-12T19:14:11.060405Z","iopub.status.idle":"2022-04-12T19:14:11.230565Z","shell.execute_reply.started":"2022-04-12T19:14:11.060354Z","shell.execute_reply":"2022-04-12T19:14:11.229799Z"},"trusted":true},"execution_count":78,"outputs":[{"execution_count":78,"output_type":"execute_result","data":{"text/plain":"   qa_id                                     question_title  \\\n0      0  What am I losing when using extension tubes in...   \n1      1  What is the distinction between a city and a s...   \n2      2  Maximum protusion length for through-hole comp...   \n3      3              Can an affidavit be used in Beit Din?   \n4      5       How do you make a binary image in Photoshop?   \n\n                                       question_body question_user_name  \\\n0  After playing around with macro photography on...               ysap   \n1  I am trying to understand what kinds of places...      russellpierce   \n2  I'm working on a PCB that has through-hole com...          Joe Baker   \n3  An affidavit, from what i understand, is basic...         Scimonster   \n4  I am trying to make a binary image. I want mor...            leigero   \n\n                                  question_user_page  \\\n0         https://photo.stackexchange.com/users/1024   \n1           https://rpg.stackexchange.com/users/8774   \n2  https://electronics.stackexchange.com/users/10157   \n3       https://judaism.stackexchange.com/users/5151   \n4  https://graphicdesign.stackexchange.com/users/...   \n\n                                              answer answer_user_name  \\\n0  I just got extension tubes, so here's the skin...           rfusca   \n1  It might be helpful to look into the definitio...     Erik Schmidt   \n2  Do you even need grooves?  We make several pro...      Dwayne Reid   \n3  Sending an \"affidavit\" it is a dispute between...    Y     e     z   \n4  Check out Image Trace in Adobe Illustrator. \\n...             q2ra   \n\n                                    answer_user_page  \\\n0         https://photo.stackexchange.com/users/1917   \n1           https://rpg.stackexchange.com/users/1871   \n2  https://electronics.stackexchange.com/users/64754   \n3       https://judaism.stackexchange.com/users/4794   \n4  https://graphicdesign.stackexchange.com/users/...   \n\n                                                 url   category  ...  \\\n0  http://photo.stackexchange.com/questions/9169/...  LIFE_ARTS  ...   \n1  http://rpg.stackexchange.com/questions/47820/w...    CULTURE  ...   \n2  http://electronics.stackexchange.com/questions...    SCIENCE  ...   \n3  http://judaism.stackexchange.com/questions/551...    CULTURE  ...   \n4  http://graphicdesign.stackexchange.com/questio...  LIFE_ARTS  ...   \n\n  question_well_written  answer_helpful  answer_level_of_information  \\\n0              1.000000        1.000000                     0.666667   \n1              0.888889        0.888889                     0.555556   \n2              0.777778        0.777778                     0.555556   \n3              0.888889        0.833333                     0.333333   \n4              1.000000        1.000000                     0.666667   \n\n   answer_plausible  answer_relevance  answer_satisfaction  \\\n0          1.000000          1.000000             0.800000   \n1          0.888889          0.888889             0.666667   \n2          1.000000          1.000000             0.666667   \n3          0.833333          1.000000             0.800000   \n4          1.000000          1.000000             0.800000   \n\n   answer_type_instructions  answer_type_procedure  \\\n0                       1.0               0.000000   \n1                       0.0               0.000000   \n2                       0.0               0.333333   \n3                       0.0               0.000000   \n4                       1.0               0.000000   \n\n   answer_type_reason_explanation  answer_well_written  \n0                        0.000000             1.000000  \n1                        0.666667             0.888889  \n2                        1.000000             0.888889  \n3                        1.000000             1.000000  \n4                        1.000000             1.000000  \n\n[5 rows x 41 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>qa_id</th>\n      <th>question_title</th>\n      <th>question_body</th>\n      <th>question_user_name</th>\n      <th>question_user_page</th>\n      <th>answer</th>\n      <th>answer_user_name</th>\n      <th>answer_user_page</th>\n      <th>url</th>\n      <th>category</th>\n      <th>...</th>\n      <th>question_well_written</th>\n      <th>answer_helpful</th>\n      <th>answer_level_of_information</th>\n      <th>answer_plausible</th>\n      <th>answer_relevance</th>\n      <th>answer_satisfaction</th>\n      <th>answer_type_instructions</th>\n      <th>answer_type_procedure</th>\n      <th>answer_type_reason_explanation</th>\n      <th>answer_well_written</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>What am I losing when using extension tubes in...</td>\n      <td>After playing around with macro photography on...</td>\n      <td>ysap</td>\n      <td>https://photo.stackexchange.com/users/1024</td>\n      <td>I just got extension tubes, so here's the skin...</td>\n      <td>rfusca</td>\n      <td>https://photo.stackexchange.com/users/1917</td>\n      <td>http://photo.stackexchange.com/questions/9169/...</td>\n      <td>LIFE_ARTS</td>\n      <td>...</td>\n      <td>1.000000</td>\n      <td>1.000000</td>\n      <td>0.666667</td>\n      <td>1.000000</td>\n      <td>1.000000</td>\n      <td>0.800000</td>\n      <td>1.0</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n      <td>1.000000</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>What is the distinction between a city and a s...</td>\n      <td>I am trying to understand what kinds of places...</td>\n      <td>russellpierce</td>\n      <td>https://rpg.stackexchange.com/users/8774</td>\n      <td>It might be helpful to look into the definitio...</td>\n      <td>Erik Schmidt</td>\n      <td>https://rpg.stackexchange.com/users/1871</td>\n      <td>http://rpg.stackexchange.com/questions/47820/w...</td>\n      <td>CULTURE</td>\n      <td>...</td>\n      <td>0.888889</td>\n      <td>0.888889</td>\n      <td>0.555556</td>\n      <td>0.888889</td>\n      <td>0.888889</td>\n      <td>0.666667</td>\n      <td>0.0</td>\n      <td>0.000000</td>\n      <td>0.666667</td>\n      <td>0.888889</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2</td>\n      <td>Maximum protusion length for through-hole comp...</td>\n      <td>I'm working on a PCB that has through-hole com...</td>\n      <td>Joe Baker</td>\n      <td>https://electronics.stackexchange.com/users/10157</td>\n      <td>Do you even need grooves?  We make several pro...</td>\n      <td>Dwayne Reid</td>\n      <td>https://electronics.stackexchange.com/users/64754</td>\n      <td>http://electronics.stackexchange.com/questions...</td>\n      <td>SCIENCE</td>\n      <td>...</td>\n      <td>0.777778</td>\n      <td>0.777778</td>\n      <td>0.555556</td>\n      <td>1.000000</td>\n      <td>1.000000</td>\n      <td>0.666667</td>\n      <td>0.0</td>\n      <td>0.333333</td>\n      <td>1.000000</td>\n      <td>0.888889</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3</td>\n      <td>Can an affidavit be used in Beit Din?</td>\n      <td>An affidavit, from what i understand, is basic...</td>\n      <td>Scimonster</td>\n      <td>https://judaism.stackexchange.com/users/5151</td>\n      <td>Sending an \"affidavit\" it is a dispute between...</td>\n      <td>Y     e     z</td>\n      <td>https://judaism.stackexchange.com/users/4794</td>\n      <td>http://judaism.stackexchange.com/questions/551...</td>\n      <td>CULTURE</td>\n      <td>...</td>\n      <td>0.888889</td>\n      <td>0.833333</td>\n      <td>0.333333</td>\n      <td>0.833333</td>\n      <td>1.000000</td>\n      <td>0.800000</td>\n      <td>0.0</td>\n      <td>0.000000</td>\n      <td>1.000000</td>\n      <td>1.000000</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>5</td>\n      <td>How do you make a binary image in Photoshop?</td>\n      <td>I am trying to make a binary image. I want mor...</td>\n      <td>leigero</td>\n      <td>https://graphicdesign.stackexchange.com/users/...</td>\n      <td>Check out Image Trace in Adobe Illustrator. \\n...</td>\n      <td>q2ra</td>\n      <td>https://graphicdesign.stackexchange.com/users/...</td>\n      <td>http://graphicdesign.stackexchange.com/questio...</td>\n      <td>LIFE_ARTS</td>\n      <td>...</td>\n      <td>1.000000</td>\n      <td>1.000000</td>\n      <td>0.666667</td>\n      <td>1.000000</td>\n      <td>1.000000</td>\n      <td>0.800000</td>\n      <td>1.0</td>\n      <td>0.000000</td>\n      <td>1.000000</td>\n      <td>1.000000</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 41 columns</p>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"test = pd.read_csv(f'{DATA_DIR}/test.csv')\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-12T19:14:11.231867Z","iopub.execute_input":"2022-04-12T19:14:11.232340Z","iopub.status.idle":"2022-04-12T19:14:11.262449Z","shell.execute_reply.started":"2022-04-12T19:14:11.232285Z","shell.execute_reply":"2022-04-12T19:14:11.261644Z"},"trusted":true},"execution_count":79,"outputs":[{"execution_count":79,"output_type":"execute_result","data":{"text/plain":"   qa_id                                     question_title  \\\n0     39  Will leaving corpses lying around upset my pri...   \n1     46         Url link to feature image in the portfolio   \n2     70  Is accuracy, recoil or bullet spread affected ...   \n3    132     Suddenly got an I/O error from my external HDD   \n4    200  Passenger Name - Flight Booking Passenger only...   \n\n                                       question_body question_user_name  \\\n0  I see questions/information online about how t...              Dylan   \n1  I am new to Wordpress. i have issue with Featu...                Anu   \n2  To experiment I started a bot game, toggled in...             Konsta   \n3  I have used my Raspberry Pi as a torrent-serve...           robbannn   \n4  I have bought Delhi-London return flights for ...               Amit   \n\n                                  question_user_page  \\\n0       https://gaming.stackexchange.com/users/64471   \n1    https://wordpress.stackexchange.com/users/72927   \n2       https://gaming.stackexchange.com/users/37545   \n3  https://raspberrypi.stackexchange.com/users/17341   \n4       https://travel.stackexchange.com/users/29089   \n\n                                              answer answer_user_name  \\\n0  There is no consequence for leaving corpses an...        Nelson868   \n1  I think it is possible with custom fields.\\n\\n...            Irina   \n2  You do not have armour in the screenshots. Thi...   Damon Smithies   \n3  Your Western Digital hard drive is disappearin...      HeatfanJohn   \n4  I called two persons who work for Saudia (tick...    Nean Der Thal   \n\n                                   answer_user_page  \\\n0      https://gaming.stackexchange.com/users/97324   \n1   https://wordpress.stackexchange.com/users/27233   \n2      https://gaming.stackexchange.com/users/70641   \n3  https://raspberrypi.stackexchange.com/users/1311   \n4      https://travel.stackexchange.com/users/10051   \n\n                                                 url    category  \\\n0  http://gaming.stackexchange.com/questions/1979...     CULTURE   \n1  http://wordpress.stackexchange.com/questions/1...  TECHNOLOGY   \n2  http://gaming.stackexchange.com/questions/2154...     CULTURE   \n3  http://raspberrypi.stackexchange.com/questions...  TECHNOLOGY   \n4  http://travel.stackexchange.com/questions/4704...     CULTURE   \n\n                            host  \n0       gaming.stackexchange.com  \n1    wordpress.stackexchange.com  \n2       gaming.stackexchange.com  \n3  raspberrypi.stackexchange.com  \n4       travel.stackexchange.com  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>qa_id</th>\n      <th>question_title</th>\n      <th>question_body</th>\n      <th>question_user_name</th>\n      <th>question_user_page</th>\n      <th>answer</th>\n      <th>answer_user_name</th>\n      <th>answer_user_page</th>\n      <th>url</th>\n      <th>category</th>\n      <th>host</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>39</td>\n      <td>Will leaving corpses lying around upset my pri...</td>\n      <td>I see questions/information online about how t...</td>\n      <td>Dylan</td>\n      <td>https://gaming.stackexchange.com/users/64471</td>\n      <td>There is no consequence for leaving corpses an...</td>\n      <td>Nelson868</td>\n      <td>https://gaming.stackexchange.com/users/97324</td>\n      <td>http://gaming.stackexchange.com/questions/1979...</td>\n      <td>CULTURE</td>\n      <td>gaming.stackexchange.com</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>46</td>\n      <td>Url link to feature image in the portfolio</td>\n      <td>I am new to Wordpress. i have issue with Featu...</td>\n      <td>Anu</td>\n      <td>https://wordpress.stackexchange.com/users/72927</td>\n      <td>I think it is possible with custom fields.\\n\\n...</td>\n      <td>Irina</td>\n      <td>https://wordpress.stackexchange.com/users/27233</td>\n      <td>http://wordpress.stackexchange.com/questions/1...</td>\n      <td>TECHNOLOGY</td>\n      <td>wordpress.stackexchange.com</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>70</td>\n      <td>Is accuracy, recoil or bullet spread affected ...</td>\n      <td>To experiment I started a bot game, toggled in...</td>\n      <td>Konsta</td>\n      <td>https://gaming.stackexchange.com/users/37545</td>\n      <td>You do not have armour in the screenshots. Thi...</td>\n      <td>Damon Smithies</td>\n      <td>https://gaming.stackexchange.com/users/70641</td>\n      <td>http://gaming.stackexchange.com/questions/2154...</td>\n      <td>CULTURE</td>\n      <td>gaming.stackexchange.com</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>132</td>\n      <td>Suddenly got an I/O error from my external HDD</td>\n      <td>I have used my Raspberry Pi as a torrent-serve...</td>\n      <td>robbannn</td>\n      <td>https://raspberrypi.stackexchange.com/users/17341</td>\n      <td>Your Western Digital hard drive is disappearin...</td>\n      <td>HeatfanJohn</td>\n      <td>https://raspberrypi.stackexchange.com/users/1311</td>\n      <td>http://raspberrypi.stackexchange.com/questions...</td>\n      <td>TECHNOLOGY</td>\n      <td>raspberrypi.stackexchange.com</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>200</td>\n      <td>Passenger Name - Flight Booking Passenger only...</td>\n      <td>I have bought Delhi-London return flights for ...</td>\n      <td>Amit</td>\n      <td>https://travel.stackexchange.com/users/29089</td>\n      <td>I called two persons who work for Saudia (tick...</td>\n      <td>Nean Der Thal</td>\n      <td>https://travel.stackexchange.com/users/10051</td>\n      <td>http://travel.stackexchange.com/questions/4704...</td>\n      <td>CULTURE</td>\n      <td>travel.stackexchange.com</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"import torch\n#import torch.utils.data as data\nfrom torchvision import datasets, models, transforms\nfrom transformers import *\nfrom sklearn.utils import shuffle\nimport random\nfrom math import floor, ceil\nfrom sklearn.model_selection import GroupKFold\n\nfrom torchtext.datasets import WikiText2\nfrom torchtext.data.utils import get_tokenizer\nfrom torchtext.vocab import build_vocab_from_iterator\n\nMAX_LEN = 512\n#MAX_Q_LEN = 250\n#MAX_A_LEN = 259\nSEP_TOKEN_ID = 102\n\nclass QuestDataset(torch.utils.data.Dataset):\n    def __init__(self, df, train_mode=True, labeled=True):\n        self.df = df\n        self.train_mode = train_mode\n        self.labeled = labeled\n        #self.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n        #self.tokenizer = BertTokenizer.from_pretrained('../input/d/datasets/abhishek/bert-base-uncased/')\n        \n        self.tokenizer = get_tokenizer('basic_english')\n\n    def __getitem__(self, index):\n        row = self.df.iloc[index]\n        token_ids, seg_ids = self.get_token_ids(row)\n        \n        self.vocab = build_vocab_from_iterator(map(tokenizer, train_iter), specials=['<unk>'])\n        self.vocab.set_default_index(vocab['<unk>'])\n        \n        if self.labeled:\n            labels = self.get_label(row)\n            return token_ids, seg_ids, labels\n        else:\n            return token_ids, seg_ids\n\n    def __len__(self):\n        return len(self.df)\n\n    def select_tokens(self, tokens, max_num):\n        if len(tokens) <= max_num:\n            return tokens\n        if self.train_mode:\n            num_remove = len(tokens) - max_num\n            remove_start = random.randint(0, len(tokens)-num_remove-1)\n            return tokens[:remove_start] + tokens[remove_start + num_remove:]\n        else:\n            return tokens[:max_num//2] + tokens[-(max_num - max_num//2):]\n\n    def trim_input(self, title, question, answer, max_sequence_length=MAX_LEN, \n                t_max_len=30, q_max_len=239, a_max_len=239):\n        t = self.tokenizer.tokenize(title)\n        q = self.tokenizer.tokenize(question)\n        a = self.tokenizer.tokenize(answer)\n\n        t_len = len(t)\n        q_len = len(q)\n        a_len = len(a)\n\n        if (t_len+q_len+a_len+4) > max_sequence_length:\n\n            if t_max_len > t_len:\n                t_new_len = t_len\n                a_max_len = a_max_len + floor((t_max_len - t_len)/2)\n                q_max_len = q_max_len + ceil((t_max_len - t_len)/2)\n            else:\n                t_new_len = t_max_len\n\n            if a_max_len > a_len:\n                a_new_len = a_len \n                q_new_len = q_max_len + (a_max_len - a_len)\n            elif q_max_len > q_len:\n                a_new_len = a_max_len + (q_max_len - q_len)\n                q_new_len = q_len\n            else:\n                a_new_len = a_max_len\n                q_new_len = q_max_len\n\n\n            if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:\n                raise ValueError(\"New sequence length should be %d, but is %d\" \n                                 % (max_sequence_length, (t_new_len+a_new_len+q_new_len+4)))\n\n            t = t[:t_new_len]\n            q = q[:q_new_len]\n            a = a[:a_new_len]\n\n        return t, q, a\n        \n    def get_token_ids(self, row):\n        t_tokens, q_tokens, a_tokens = self.trim_input(row.question_title, row.question_body, row.answer)\n\n        tokens = ['[CLS]'] + t_tokens + ['[SEP]'] + q_tokens + ['[SEP]'] + a_tokens + ['[SEP]']\n        token_ids = self.tokenizer.convert_tokens_to_ids(tokens)\n        if len(token_ids) < MAX_LEN:\n            token_ids += [0] * (MAX_LEN - len(token_ids))\n        ids = torch.tensor(token_ids)\n        seg_ids = self.get_seg_ids(ids)\n        return ids, seg_ids\n    \n    def get_seg_ids(self, ids):\n        seg_ids = torch.zeros_like(ids)\n        seg_idx = 0\n        first_sep = True\n        for i, e in enumerate(ids):\n            seg_ids[i] = seg_idx\n            if e == SEP_TOKEN_ID:\n                if first_sep:\n                    first_sep = False\n                else:\n                    seg_idx = 1\n        pad_idx = torch.nonzero(ids == 0)\n        seg_ids[pad_idx] = 0\n\n        return seg_ids\n\n    def get_label(self, row):\n        print(row[target_columns].values)\n        return torch.tensor(row[target_columns].values.astype(np.float32))\n\n    def collate_fn(self, batch):\n        token_ids = torch.stack([x[0] for x in batch])\n        seg_ids = torch.stack([x[1] for x in batch])\n    \n        if self.labeled:\n            labels = torch.stack([x[2] for x in batch])\n            return token_ids, seg_ids, labels\n        else:\n            return token_ids, seg_ids\n\ndef get_test_loader(batch_size=4):\n    df = pd.read_csv(f'{DATA_DIR}/test.csv')\n    ds_test = QuestDataset(df, train_mode=False, labeled=False)\n    loader = torch.utils.data.DataLoader(ds_test, batch_size=batch_size, shuffle=False, num_workers=0, collate_fn=ds_test.collate_fn, drop_last=False)\n    loader.num = len(df)\n    \n    return loader\n        \ndef get_train_val_loaders(batch_size=4, val_batch_size=4, ifold=0):\n    df = pd.read_csv(f'{DATA_DIR}/train.csv')\n    df = shuffle(df, random_state=1234)\n    #split_index = int(len(df) * (1-val_percent))\n    gkf = GroupKFold(n_splits=5).split(X=df.question_body, groups=df.question_body)\n    for fold, (train_idx, valid_idx) in enumerate(gkf):\n        if fold == ifold:\n            df_train = df.iloc[train_idx]\n            df_val = df.iloc[valid_idx]\n            break\n\n    #print(df_val.head())\n    #df_train = df[:split_index]\n    #df_val = df[split_index:]\n\n    print(df_train.shape)\n    print(df_val.shape)\n\n    ds_train = QuestDataset(df_train)\n    train_loader = torch.utils.data.DataLoader(ds_train, batch_size=batch_size, shuffle=True, num_workers=0, collate_fn=ds_train.collate_fn, drop_last=True)\n    train_loader.num = len(df_train)\n\n    ds_val = QuestDataset(df_val, train_mode=False)\n    val_loader = torch.utils.data.DataLoader(ds_val, batch_size=val_batch_size, shuffle=False, num_workers=0, collate_fn=ds_val.collate_fn, drop_last=False)\n    val_loader.num = len(df_val)\n    val_loader.df = df_val\n\n    return train_loader, val_loader\n\ndef test_train_loader():\n    loader, _ = get_train_val_loaders(4, 4, 1)\n    for ids, seg_ids, labels in loader:\n        print(\"ids: \",ids)\n        print(\"seg_ids: \",seg_ids.numpy())\n        print(\"labels: \",labels)\n        break\ndef test_test_loader():\n    loader = get_test_loader(4)\n    for ids, seg_ids in loader:\n        print(\"ids: \",ids)\n        print(\"seg_ids: \",seg_ids)\n        break","metadata":{"execution":{"iopub.status.busy":"2022-04-12T19:14:11.264350Z","iopub.execute_input":"2022-04-12T19:14:11.264876Z","iopub.status.idle":"2022-04-12T19:14:11.309842Z","shell.execute_reply.started":"2022-04-12T19:14:11.264669Z","shell.execute_reply":"2022-04-12T19:14:11.308663Z"},"trusted":true},"execution_count":80,"outputs":[]},{"cell_type":"code","source":"pwd","metadata":{"execution":{"iopub.status.busy":"2022-04-12T19:14:11.311286Z","iopub.execute_input":"2022-04-12T19:14:11.311736Z","iopub.status.idle":"2022-04-12T19:14:11.323542Z","shell.execute_reply.started":"2022-04-12T19:14:11.311545Z","shell.execute_reply":"2022-04-12T19:14:11.322753Z"},"trusted":true},"execution_count":81,"outputs":[{"execution_count":81,"output_type":"execute_result","data":{"text/plain":"'/kaggle/working'"},"metadata":{}}]},{"cell_type":"code","source":"ls ../input/d/datasets/abhishek/bert-base-uncased","metadata":{"execution":{"iopub.status.busy":"2022-04-12T19:14:11.326794Z","iopub.execute_input":"2022-04-12T19:14:11.327184Z","iopub.status.idle":"2022-04-12T19:14:12.027928Z","shell.execute_reply.started":"2022-04-12T19:14:11.327001Z","shell.execute_reply":"2022-04-12T19:14:12.026661Z"},"trusted":true},"execution_count":82,"outputs":[{"name":"stdout","text":"config.json  pytorch_model.bin  vocab.txt\n","output_type":"stream"}]},{"cell_type":"markdown","source":"## Build Model","metadata":{}},{"cell_type":"code","source":"import math\nfrom typing import Tuple\n\nimport torch\nfrom torch import nn, Tensor\nimport torch.nn.functional as F\nfrom torch.nn import TransformerEncoder, TransformerEncoderLayer\nfrom torch.utils.data import dataset\n\nclass TransformerModel(nn.Module):\n\n    def __init__(self, ntoken: int, d_model: int, nhead: int, d_hid: int,\n                 nlayers: int, dropout: float = 0.5):\n        super().__init__()\n        self.model_type = 'Transformer'\n        self.pos_encoder = PositionalEncoding(d_model, dropout)\n        encoder_layers = TransformerEncoderLayer(d_model, nhead, d_hid, dropout)\n        self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)\n        self.encoder = nn.Embedding(ntoken, d_model)\n        self.d_model = d_model\n        self.decoder = nn.Linear(d_model, ntoken)\n\n        self.init_weights()\n\n    def init_weights(self) -> None:\n        initrange = 0.1\n        self.encoder.weight.data.uniform_(-initrange, initrange)\n        self.decoder.bias.data.zero_()\n        self.decoder.weight.data.uniform_(-initrange, initrange)\n\n    def forward(self, src: Tensor, src_mask: Tensor) -> Tensor:\n        \"\"\"\n        Args:\n            src: Tensor, shape [seq_len, batch_size]\n            src_mask: Tensor, shape [seq_len, seq_len]\n\n        Returns:\n            output Tensor of shape [seq_len, batch_size, ntoken]\n        \"\"\"\n        src = self.encoder(src) * math.sqrt(self.d_model)\n        src = self.pos_encoder(src)\n        output = self.transformer_encoder(src, src_mask)\n        output = self.decoder(output)\n        return output\n\n\ndef generate_square_subsequent_mask(sz: int) -> Tensor:\n    \"\"\"Generates an upper-triangular matrix of -inf, with zeros on diag.\"\"\"\n    return torch.triu(torch.ones(sz, sz) * float('-inf'), diagonal=1)","metadata":{"execution":{"iopub.status.busy":"2022-04-12T19:14:12.031741Z","iopub.execute_input":"2022-04-12T19:14:12.032020Z","iopub.status.idle":"2022-04-12T19:14:12.046545Z","shell.execute_reply.started":"2022-04-12T19:14:12.031970Z","shell.execute_reply":"2022-04-12T19:14:12.045667Z"},"trusted":true},"execution_count":83,"outputs":[]},{"cell_type":"code","source":"class PositionalEncoding(nn.Module):\n\n    def __init__(self, d_model: int, dropout: float = 0.1, max_len: int = 5000):\n        super().__init__()\n        self.dropout = nn.Dropout(p=dropout)\n\n        position = torch.arange(max_len).unsqueeze(1)\n        div_term = torch.exp(torch.arange(0, d_model, 2) * (-math.log(10000.0) / d_model))\n        pe = torch.zeros(max_len, 1, d_model)\n        pe[:, 0, 0::2] = torch.sin(position * div_term)\n        pe[:, 0, 1::2] = torch.cos(position * div_term)\n        self.register_buffer('pe', pe)\n\n    def forward(self, x: Tensor) -> Tensor:\n        \"\"\"\n        Args:\n            x: Tensor, shape [seq_len, batch_size, embedding_dim]\n        \"\"\"\n        x = x + self.pe[:x.size(0)]\n        return self.dropout(x)","metadata":{"execution":{"iopub.status.busy":"2022-04-12T19:14:12.048140Z","iopub.execute_input":"2022-04-12T19:14:12.048799Z","iopub.status.idle":"2022-04-12T19:14:12.059868Z","shell.execute_reply.started":"2022-04-12T19:14:12.048748Z","shell.execute_reply":"2022-04-12T19:14:12.059153Z"},"trusted":true},"execution_count":84,"outputs":[]},{"cell_type":"code","source":"from transformers import *\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\n\nclass QuestModel(nn.Module):\n    def __init__(self,ids, n_classes=30):\n        super(QuestModel, self).__init__()\n        self.model_name = 'QuestModel'\n        #self.bert_model = BertModel.from_pretrained('../input/d/datasets/abhishek/bert-base-uncased/')\n        \n        #ntokens = len(vocab)  # size of vocabulary\n        ntokens = len(ids)  # size of vocabulary\n        emsize = 200  # embedding dimension\n        d_hid = 200  # dimension of the feedforward network model in nn.TransformerEncoder\n        nlayers = 2  # number of nn.TransformerEncoderLayer in nn.TransformerEncoder\n        nhead = 2  # number of heads in nn.MultiheadAttention\n        dropout = 0.2  # dropout probability\n        \n        self.transformer_model=TransformerModel(ntokens, emsize, nhead, d_hid, nlayers, dropout)\n        \n        self.fc = nn.Linear(768, n_classes)\n        \n    def forward(self, ids, seg_ids):\n        attention_mask = (ids > 0)\n        #layers, pool_out = self.bert_model(input_ids=ids, token_type_ids=seg_ids, attention_mask=attention_mask)\n        \n        out =  self.transformer_model(ids, seg_ids)\n        \n        #print(layers[-1][0].size())\n        #print(pool_out.size())\n\n        #out = F.dropout(layers[-1][:, 0, :], p=0.2, training=self.training)\n        #out =  F.dropout(pool_out, p=0.2, training=self.training)\n        \n        logit = self.fc(out)\n        return logit\n    \ndef test_model():\n    x = torch.tensor([[1,2,3,4,5, 0, 0], [1,2,3,4,5, 0, 0]])\n    seg_ids = torch.tensor([[0,0,0,0,0, 0, 0], [0,0,0,0,0, 0, 0]])\n    model = QuestModel(x)\n\n    y = model(x, seg_ids)\n    print(y)","metadata":{"execution":{"iopub.status.busy":"2022-04-12T19:15:31.495611Z","iopub.execute_input":"2022-04-12T19:15:31.495920Z","iopub.status.idle":"2022-04-12T19:15:31.508200Z","shell.execute_reply.started":"2022-04-12T19:15:31.495870Z","shell.execute_reply":"2022-04-12T19:15:31.507011Z"},"trusted":true},"execution_count":91,"outputs":[]},{"cell_type":"code","source":"!ls ../input","metadata":{"execution":{"iopub.status.busy":"2022-04-12T19:15:31.789359Z","iopub.execute_input":"2022-04-12T19:15:31.789657Z","iopub.status.idle":"2022-04-12T19:15:32.511787Z","shell.execute_reply.started":"2022-04-12T19:15:31.789605Z","shell.execute_reply":"2022-04-12T19:15:32.510774Z"},"trusted":true},"execution_count":92,"outputs":[{"name":"stdout","text":"d  google-quest-challenge  quest-models  sacremoses  transformers\n","output_type":"stream"}]},{"cell_type":"code","source":"def create_model(ids):\n    model = QuestModel(ids)\n    #model.load_state_dict(torch.load(model_file))\n    model = model.cuda()\n    #model = DataParallel(model)\n    return model\n\ndef create_models(ids):\n    models = []\n    for i in range(5):\n        model = create_model(ids)\n        model.eval()\n        models.append(model)\n    return models","metadata":{"execution":{"iopub.status.busy":"2022-04-12T19:16:14.000359Z","iopub.execute_input":"2022-04-12T19:16:14.000673Z","iopub.status.idle":"2022-04-12T19:16:14.006737Z","shell.execute_reply.started":"2022-04-12T19:16:14.000623Z","shell.execute_reply":"2022-04-12T19:16:14.005990Z"},"trusted":true},"execution_count":102,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\nimport torch\ndef predict(models, test_loader):\n    all_scores = []\n    with torch.no_grad():\n        for ids, seg_ids in tqdm(test_loader, total=test_loader.num // test_loader.batch_size):\n            ids, seg_ids = ids.cuda(), seg_ids.cuda()\n            scores = []\n            for model in models:\n                outputs = torch.sigmoid(model(ids, seg_ids)).cpu()\n                scores.append(outputs)\n            all_scores.append(torch.mean(torch.stack(scores), 0))\n\n    all_scores = torch.cat(all_scores, 0).numpy()\n    \n    return all_scores","metadata":{"execution":{"iopub.status.busy":"2022-04-12T19:16:14.293521Z","iopub.execute_input":"2022-04-12T19:16:14.293853Z","iopub.status.idle":"2022-04-12T19:16:14.301922Z","shell.execute_reply.started":"2022-04-12T19:16:14.293784Z","shell.execute_reply":"2022-04-12T19:16:14.301044Z"},"trusted":true},"execution_count":103,"outputs":[]},{"cell_type":"code","source":"test_loader = get_test_loader(batch_size=32)","metadata":{"execution":{"iopub.status.busy":"2022-04-12T19:16:14.595836Z","iopub.execute_input":"2022-04-12T19:16:14.596137Z","iopub.status.idle":"2022-04-12T19:16:14.618998Z","shell.execute_reply.started":"2022-04-12T19:16:14.596083Z","shell.execute_reply":"2022-04-12T19:16:14.618345Z"},"trusted":true},"execution_count":104,"outputs":[]},{"cell_type":"code","source":"models = create_models(test_loader)","metadata":{"execution":{"iopub.status.busy":"2022-04-12T19:16:14.999695Z","iopub.execute_input":"2022-04-12T19:16:15.000093Z","iopub.status.idle":"2022-04-12T19:16:15.074916Z","shell.execute_reply.started":"2022-04-12T19:16:15.000016Z","shell.execute_reply":"2022-04-12T19:16:15.074075Z"},"trusted":true},"execution_count":105,"outputs":[]},{"cell_type":"code","source":"preds = predict(models, test_loader)","metadata":{"execution":{"iopub.status.busy":"2022-04-12T19:16:15.291215Z","iopub.execute_input":"2022-04-12T19:16:15.291529Z","iopub.status.idle":"2022-04-12T19:16:15.382394Z","shell.execute_reply.started":"2022-04-12T19:16:15.291470Z","shell.execute_reply":"2022-04-12T19:16:15.381078Z"},"trusted":true},"execution_count":106,"outputs":[{"name":"stderr","text":"\n  0%|          | 0/14 [00:00<?, ?it/s]\u001b[A","output_type":"stream"},{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)","\u001b[0;32m<ipython-input-106-67c79b17573e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpreds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_loader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;32m<ipython-input-103-1480f50fb575>\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(models, test_loader)\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mall_scores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0mids\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mseg_ids\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtotal\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtest_loader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum\u001b[0m \u001b[0;34m//\u001b[0m \u001b[0mtest_loader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m             \u001b[0mids\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mseg_ids\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mids\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mseg_ids\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m             \u001b[0mscores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tqdm/std.py\u001b[0m in \u001b[0;36m__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1079\u001b[0m             \"\"\"), fp_write=getattr(self.fp, 'write', sys.stderr.write))\n\u001b[1;32m   1080\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1081\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miterable\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1082\u001b[0m             \u001b[0;32myield\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1083\u001b[0m             \u001b[0;31m# Update and possibly print the progressbar.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    344\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__next__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    345\u001b[0m         \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 346\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_fetcher\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    347\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pin_memory\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    348\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36mfetch\u001b[0;34m(self, possibly_batched_index)\u001b[0m\n\u001b[1;32m     42\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauto_collation\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 44\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     45\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     42\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauto_collation\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 44\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     45\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m<ipython-input-80-a4ebd1bc794b>\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, index)\u001b[0m\n\u001b[1;32m     29\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__getitem__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m         \u001b[0mrow\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 31\u001b[0;31m         \u001b[0mtoken_ids\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mseg_ids\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_token_ids\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     32\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocab\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbuild_vocab_from_iterator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtokenizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_iter\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mspecials\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'<unk>'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m<ipython-input-80-a4ebd1bc794b>\u001b[0m in \u001b[0;36mget_token_ids\u001b[0;34m(self, row)\u001b[0m\n\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget_token_ids\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrow\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 96\u001b[0;31m         \u001b[0mt_tokens\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mq_tokens\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ma_tokens\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrim_input\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mquestion_title\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mquestion_body\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0manswer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     97\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     98\u001b[0m         \u001b[0mtokens\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'[CLS]'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mt_tokens\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'[SEP]'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mq_tokens\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'[SEP]'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0ma_tokens\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'[SEP]'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m<ipython-input-80-a4ebd1bc794b>\u001b[0m in \u001b[0;36mtrim_input\u001b[0;34m(self, title, question, answer, max_sequence_length, t_max_len, q_max_len, a_max_len)\u001b[0m\n\u001b[1;32m     55\u001b[0m     def trim_input(self, title, question, answer, max_sequence_length=MAX_LEN, \n\u001b[1;32m     56\u001b[0m                 t_max_len=30, q_max_len=239, a_max_len=239):\n\u001b[0;32m---> 57\u001b[0;31m         \u001b[0mt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtokenize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtitle\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     58\u001b[0m         \u001b[0mq\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtokenize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquestion\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m         \u001b[0ma\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtokenize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0manswer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mAttributeError\u001b[0m: 'function' object has no attribute 'tokenize'"],"ename":"AttributeError","evalue":"'function' object has no attribute 'tokenize'","output_type":"error"}]},{"cell_type":"code","source":"preds[:1]","metadata":{"execution":{"iopub.status.busy":"2022-04-12T19:16:15.584336Z","iopub.execute_input":"2022-04-12T19:16:15.584634Z","iopub.status.idle":"2022-04-12T19:16:15.611245Z","shell.execute_reply.started":"2022-04-12T19:16:15.584580Z","shell.execute_reply":"2022-04-12T19:16:15.610271Z"},"trusted":true},"execution_count":107,"outputs":[{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)","\u001b[0;32m<ipython-input-107-e99dae2e2117>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpreds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;31mNameError\u001b[0m: name 'preds' is not defined"],"ename":"NameError","evalue":"name 'preds' is not defined","output_type":"error"}]},{"cell_type":"code","source":"preds","metadata":{"execution":{"iopub.status.busy":"2022-04-12T19:16:15.932519Z","iopub.execute_input":"2022-04-12T19:16:15.932833Z","iopub.status.idle":"2022-04-12T19:16:15.958457Z","shell.execute_reply.started":"2022-04-12T19:16:15.932779Z","shell.execute_reply":"2022-04-12T19:16:15.957098Z"},"trusted":true},"execution_count":108,"outputs":[{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)","\u001b[0;32m<ipython-input-108-c66c8509e20f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpreds\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;31mNameError\u001b[0m: name 'preds' is not defined"],"ename":"NameError","evalue":"name 'preds' is not defined","output_type":"error"}]},{"cell_type":"markdown","source":"### Generate Submission","metadata":{}},{"cell_type":"code","source":"test_loss = evaluate(best_model, test_data)\ntest_ppl = math.exp(test_loss)\nprint('=' * 89)\nprint(f'| End of training | test loss {test_loss:5.2f} | '\n      f'test ppl {test_ppl:8.2f}')\nprint('=' * 89)","metadata":{"execution":{"iopub.status.busy":"2022-04-12T19:16:16.060632Z","iopub.execute_input":"2022-04-12T19:16:16.060925Z","iopub.status.idle":"2022-04-12T19:16:16.086357Z","shell.execute_reply.started":"2022-04-12T19:16:16.060876Z","shell.execute_reply":"2022-04-12T19:16:16.085265Z"},"trusted":true},"execution_count":109,"outputs":[{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)","\u001b[0;32m<ipython-input-109-1e5742a8615b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtest_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbest_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mtest_ppl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'='\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;36m89\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m print(f'| End of training | test loss {test_loss:5.2f} | '\n\u001b[1;32m      5\u001b[0m       f'test ppl {test_ppl:8.2f}')\n","\u001b[0;31mNameError\u001b[0m: name 'evaluate' is not defined"],"ename":"NameError","evalue":"name 'evaluate' is not defined","output_type":"error"}]},{"cell_type":"code","source":"sub[target_columns] = preds","metadata":{"execution":{"iopub.status.busy":"2022-04-12T19:16:16.087585Z","iopub.status.idle":"2022-04-12T19:16:16.088314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-04-12T19:16:16.190614Z","iopub.execute_input":"2022-04-12T19:16:16.190904Z","iopub.status.idle":"2022-04-12T19:16:16.221485Z","shell.execute_reply.started":"2022-04-12T19:16:16.190852Z","shell.execute_reply":"2022-04-12T19:16:16.220723Z"},"trusted":true},"execution_count":110,"outputs":[]},{"cell_type":"code","source":"sub.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-12T19:16:16.345820Z","iopub.execute_input":"2022-04-12T19:16:16.346136Z","iopub.status.idle":"2022-04-12T19:16:16.375888Z","shell.execute_reply.started":"2022-04-12T19:16:16.346085Z","shell.execute_reply":"2022-04-12T19:16:16.375129Z"},"trusted":true},"execution_count":111,"outputs":[{"execution_count":111,"output_type":"execute_result","data":{"text/plain":"   qa_id  question_asker_intent_understanding  question_body_critical  \\\n0     39                              0.00308                 0.00308   \n1     46                              0.00448                 0.00448   \n2     70                              0.00673                 0.00673   \n3    132                              0.01401                 0.01401   \n4    200                              0.02074                 0.02074   \n\n   question_conversational  question_expect_short_answer  \\\n0                  0.00308                       0.00308   \n1                  0.00448                       0.00448   \n2                  0.00673                       0.00673   \n3                  0.01401                       0.01401   \n4                  0.02074                       0.02074   \n\n   question_fact_seeking  question_has_commonly_accepted_answer  \\\n0                0.00308                                0.00308   \n1                0.00448                                0.00448   \n2                0.00673                                0.00673   \n3                0.01401                                0.01401   \n4                0.02074                                0.02074   \n\n   question_interestingness_others  question_interestingness_self  \\\n0                          0.00308                        0.00308   \n1                          0.00448                        0.00448   \n2                          0.00673                        0.00673   \n3                          0.01401                        0.01401   \n4                          0.02074                        0.02074   \n\n   question_multi_intent  ...  question_well_written  answer_helpful  \\\n0                0.00308  ...                0.00308         0.00308   \n1                0.00448  ...                0.00448         0.00448   \n2                0.00673  ...                0.00673         0.00673   \n3                0.01401  ...                0.01401         0.01401   \n4                0.02074  ...                0.02074         0.02074   \n\n   answer_level_of_information  answer_plausible  answer_relevance  \\\n0                      0.00308           0.00308           0.00308   \n1                      0.00448           0.00448           0.00448   \n2                      0.00673           0.00673           0.00673   \n3                      0.01401           0.01401           0.01401   \n4                      0.02074           0.02074           0.02074   \n\n   answer_satisfaction  answer_type_instructions  answer_type_procedure  \\\n0              0.00308                   0.00308                0.00308   \n1              0.00448                   0.00448                0.00448   \n2              0.00673                   0.00673                0.00673   \n3              0.01401                   0.01401                0.01401   \n4              0.02074                   0.02074                0.02074   \n\n   answer_type_reason_explanation  answer_well_written  \n0                         0.00308              0.00308  \n1                         0.00448              0.00448  \n2                         0.00673              0.00673  \n3                         0.01401              0.01401  \n4                         0.02074              0.02074  \n\n[5 rows x 31 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>qa_id</th>\n      <th>question_asker_intent_understanding</th>\n      <th>question_body_critical</th>\n      <th>question_conversational</th>\n      <th>question_expect_short_answer</th>\n      <th>question_fact_seeking</th>\n      <th>question_has_commonly_accepted_answer</th>\n      <th>question_interestingness_others</th>\n      <th>question_interestingness_self</th>\n      <th>question_multi_intent</th>\n      <th>...</th>\n      <th>question_well_written</th>\n      <th>answer_helpful</th>\n      <th>answer_level_of_information</th>\n      <th>answer_plausible</th>\n      <th>answer_relevance</th>\n      <th>answer_satisfaction</th>\n      <th>answer_type_instructions</th>\n      <th>answer_type_procedure</th>\n      <th>answer_type_reason_explanation</th>\n      <th>answer_well_written</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>39</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>...</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n      <td>0.00308</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>46</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>...</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n      <td>0.00448</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>70</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>...</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n      <td>0.00673</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>132</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>...</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n      <td>0.01401</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>200</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>...</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n      <td>0.02074</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 31 columns</p>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}