llama-7b-finetuned-llm-science-exam.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !pip install -q -U git+https://github.com/lvwerra/trl.git\n# !pip install -q -U bitsandbytes\n# !pip install -q -U git+https://github.com/huggingface/transformers.git\n# !pip install -q -U git+https://github.com/huggingface/peft.git\n# !pip install -q -U git+https://github.com/huggingface/accelerate.git\n# !pip install -q langchain\n# !pip install -q einops","metadata":{"execution":{"iopub.status.busy":"2023-08-07T12:19:32.672546Z","iopub.execute_input":"2023-08-07T12:19:32.672930Z","iopub.status.idle":"2023-08-07T12:19:32.679376Z","shell.execute_reply.started":"2023-08-07T12:19:32.672889Z","shell.execute_reply":"2023-08-07T12:19:32.678289Z"},"trusted":true},"execution_count":1,"outputs":[]},{"cell_type":"code","source":"!pip install accelerate==0.21.0 peft==0.4.0 bitsandbytes==0.40.2 \\\n            transformers==4.30.2 trl==0.4.7 langchain einops","metadata":{"execution":{"iopub.status.busy":"2023-08-07T12:19:32.684457Z","iopub.execute_input":"2023-08-07T12:19:32.684748Z","iopub.status.idle":"2023-08-07T12:19:45.586653Z","shell.execute_reply.started":"2023-08-07T12:19:32.684723Z","shell.execute_reply":"2023-08-07T12:19:45.585438Z"},"trusted":true},"execution_count":2,"outputs":[{"name":"stdout","text":"Requirement already satisfied: accelerate==0.21.0 in /opt/conda/lib/python3.10/site-packages (0.21.0)\nRequirement already satisfied: peft==0.4.0 in /opt/conda/lib/python3.10/site-packages (0.4.0)\nRequirement already satisfied: bitsandbytes==0.40.2 in /opt/conda/lib/python3.10/site-packages (0.40.2)\nRequirement already satisfied: transformers==4.30.2 in /opt/conda/lib/python3.10/site-packages (4.30.2)\nRequirement already satisfied: trl==0.4.7 in /opt/conda/lib/python3.10/site-packages (0.4.7)\nRequirement already satisfied: langchain in /opt/conda/lib/python3.10/site-packages (0.0.254)\nRequirement already satisfied: einops in /opt/conda/lib/python3.10/site-packages (0.6.1)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from accelerate==0.21.0) (1.23.5)\nRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from accelerate==0.21.0) (21.3)\nRequirement already satisfied: psutil in /opt/conda/lib/python3.10/site-packages (from accelerate==0.21.0) (5.9.3)\nRequirement already satisfied: pyyaml in /opt/conda/lib/python3.10/site-packages (from accelerate==0.21.0) (6.0)\nRequirement already satisfied: torch>=1.10.0 in /opt/conda/lib/python3.10/site-packages (from accelerate==0.21.0) (2.0.0)\nRequirement already satisfied: safetensors in /opt/conda/lib/python3.10/site-packages (from peft==0.4.0) (0.3.1)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.2) (3.12.2)\nRequirement already satisfied: huggingface-hub<1.0,>=0.14.1 in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.2) (0.16.4)\nRequirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.2) (2023.6.3)\nRequirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.2) (2.31.0)\nRequirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.2) (0.13.3)\nRequirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.10/site-packages (from transformers==4.30.2) (4.65.0)\nRequirement already satisfied: datasets in /opt/conda/lib/python3.10/site-packages (from trl==0.4.7) (2.1.0)\nRequirement already satisfied: SQLAlchemy<3,>=1.4 in /opt/conda/lib/python3.10/site-packages (from langchain) (2.0.17)\nRequirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /opt/conda/lib/python3.10/site-packages (from langchain) (3.8.4)\nRequirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /opt/conda/lib/python3.10/site-packages (from langchain) (4.0.2)\nRequirement already satisfied: dataclasses-json<0.6.0,>=0.5.7 in /opt/conda/lib/python3.10/site-packages (from langchain) (0.5.9)\nRequirement already satisfied: langsmith<0.1.0,>=0.0.11 in /opt/conda/lib/python3.10/site-packages (from langchain) (0.0.19)\nRequirement already satisfied: numexpr<3.0.0,>=2.8.4 in /opt/conda/lib/python3.10/site-packages (from langchain) (2.8.4)\nRequirement already satisfied: openapi-schema-pydantic<2.0,>=1.2 in /opt/conda/lib/python3.10/site-packages (from langchain) (1.2.4)\nRequirement already satisfied: pydantic<2,>=1 in /opt/conda/lib/python3.10/site-packages (from langchain) (1.10.10)\nRequirement already satisfied: tenacity<9.0.0,>=8.1.0 in /opt/conda/lib/python3.10/site-packages (from langchain) (8.2.2)\nRequirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (23.1.0)\nRequirement already satisfied: charset-normalizer<4.0,>=2.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (3.1.0)\nRequirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (6.0.4)\nRequirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.9.2)\nRequirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.3.3)\nRequirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.3.1)\nRequirement already satisfied: marshmallow<4.0.0,>=3.3.0 in /opt/conda/lib/python3.10/site-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain) (3.19.0)\nRequirement already satisfied: marshmallow-enum<2.0.0,>=1.5.1 in /opt/conda/lib/python3.10/site-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain) (1.5.1)\nRequirement already satisfied: typing-inspect>=0.4.0 in /opt/conda/lib/python3.10/site-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain) (0.9.0)\nRequirement already satisfied: fsspec in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.14.1->transformers==4.30.2) (2023.6.0)\nRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.14.1->transformers==4.30.2) (4.6.3)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.0->accelerate==0.21.0) (3.0.9)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->transformers==4.30.2) (3.4)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->transformers==4.30.2) (1.26.15)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->transformers==4.30.2) (2023.5.7)\nRequirement already satisfied: greenlet!=0.4.17 in /opt/conda/lib/python3.10/site-packages (from SQLAlchemy<3,>=1.4->langchain) (2.0.2)\nRequirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate==0.21.0) (1.12)\nRequirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate==0.21.0) (3.1)\nRequirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate==0.21.0) (3.1.2)\nRequirement already satisfied: pyarrow>=5.0.0 in /opt/conda/lib/python3.10/site-packages (from datasets->trl==0.4.7) (11.0.0)\nRequirement already satisfied: dill in /opt/conda/lib/python3.10/site-packages (from datasets->trl==0.4.7) (0.3.6)\nRequirement already satisfied: pandas in /opt/conda/lib/python3.10/site-packages (from datasets->trl==0.4.7) (1.5.3)\nRequirement already satisfied: xxhash in /opt/conda/lib/python3.10/site-packages (from datasets->trl==0.4.7) (3.2.0)\nRequirement already satisfied: multiprocess in /opt/conda/lib/python3.10/site-packages (from datasets->trl==0.4.7) (0.70.14)\nRequirement already satisfied: responses<0.19 in /opt/conda/lib/python3.10/site-packages (from datasets->trl==0.4.7) (0.18.0)\nRequirement already satisfied: mypy-extensions>=0.3.0 in /opt/conda/lib/python3.10/site-packages (from typing-inspect>=0.4.0->dataclasses-json<0.6.0,>=0.5.7->langchain) (1.0.0)\nRequirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch>=1.10.0->accelerate==0.21.0) (2.1.3)\nRequirement already satisfied: python-dateutil>=2.8.1 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets->trl==0.4.7) (2.8.2)\nRequirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets->trl==0.4.7) (2023.3)\nRequirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.10.0->accelerate==0.21.0) (1.3.0)\nRequirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil>=2.8.1->pandas->datasets->trl==0.4.7) (1.16.0)\n","output_type":"stream"}]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport torch\nimport transformers\n\nfrom datasets import load_dataset\nfrom torch import nn \nfrom transformers import AutoModelForCausalLM, BitsAndBytesConfig, AutoTokenizer, TrainingArguments\nfrom peft import LoraConfig, PeftModel, PeftConfig\nfrom trl import SFTTrainer, DataCollatorForCompletionOnlyLM\nfrom langchain.prompts import PromptTemplate\nfrom IPython.display import Markdown, display\nfrom tqdm.notebook import tqdm","metadata":{"execution":{"iopub.status.busy":"2023-08-07T12:19:45.590039Z","iopub.execute_input":"2023-08-07T12:19:45.590857Z","iopub.status.idle":"2023-08-07T12:19:58.380893Z","shell.execute_reply.started":"2023-08-07T12:19:45.590824Z","shell.execute_reply":"2023-08-07T12:19:58.379908Z"},"trusted":true},"execution_count":3,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.23.5\n  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/__init__.py:98: UserWarning: unable to load libtensorflow_io_plugins.so: unable to open file: libtensorflow_io_plugins.so, from paths: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io_plugins.so']\ncaused by: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io_plugins.so: undefined symbol: _ZN3tsl6StatusC1EN10tensorflow5error4CodeESt17basic_string_viewIcSt11char_traitsIcEENS_14SourceLocationE']\n  warnings.warn(f\"unable to load libtensorflow_io_plugins.so: {e}\")\n/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/__init__.py:104: UserWarning: file system plugins are not loaded: unable to open file: libtensorflow_io.so, from paths: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io.so']\ncaused by: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io.so: undefined symbol: _ZTVN10tensorflow13GcsFileSystemE']\n  warnings.warn(f\"file system plugins are not loaded: {e}\")\n","output_type":"stream"}]},{"cell_type":"code","source":"data = pd.read_csv(\"/kaggle/input/llm-science-dataset/High Quality Dataset.csv\")\n\ndef clean(text):\n    return text.strip()\n\nfor column in data.columns:\n    if column != \"id\":\n        data[column] = data[column].astype(\"string\")\n        data[column] = data[column].apply(clean)\n\ndata = data.sample(len(data), random_state=2023)\ndata[\"id\"] = range(len(data))\ndata.reset_index(drop=True, inplace=True)\ndata.head()","metadata":{"execution":{"iopub.status.busy":"2023-08-07T12:19:58.382236Z","iopub.execute_input":"2023-08-07T12:19:58.383212Z","iopub.status.idle":"2023-08-07T12:19:58.523056Z","shell.execute_reply.started":"2023-08-07T12:19:58.383171Z","shell.execute_reply":"2023-08-07T12:19:58.521942Z"},"trusted":true},"execution_count":4,"outputs":[{"execution_count":4,"output_type":"execute_result","data":{"text/plain":"   id                                             prompt  \\\n0   0  What are the applications of confrontation ana...   \n1   1  How did the English dog breeds and those indig...   \n2   2                                What is Metelsdorf?   \n3   3  What is the temperature range at which ice Ic,...   \n4   4  How did the Saint Lucia Kings perform in the f...   \n\n                                                   A  \\\n0                     Military strategy and planning   \n1  The English dog breeds and those indigenous to...   \n2  Metelsdorf is a town in the Nordwestmecklenbur...   \n3                     Between -240 and -130 kelvins.   \n4  The Saint Lucia Kings won 6 games out of 14 in...   \n\n                                                   B  \\\n0                          Political decision-making   \n1  The English dog breeds had no influence on the...   \n2  Metelsdorf is a village in the Nordwestmecklen...   \n3            Between 220 and 240 degrees Fahrenheit.   \n4  The Saint Lucia Kings won all 14 games in the ...   \n\n                                                   C  \\\n0                          Legal disputes resolution   \n1  The English dog breeds had a major influence o...   \n2  Metelsdorf is a municipality in the Nordwestme...   \n3             Between -273 and -100 degrees Celsius.   \n4  The Saint Lucia Kings did not win any games in...   \n\n                                                   D  \\\n0                                 Financial analysis   \n1  The English dog breeds primarily influenced th...   \n2  Metelsdorf is a city in the Nordwestmecklenbur...   \n3       Between -33 degrees Celsius and 240 kelvins.   \n4  The Saint Lucia Kings won 4 games out of 14 in...   \n\n                                                   E answer  \n0                                   All of the above      E  \n1  The English dog breeds and those indigenous to...      E  \n2  Metelsdorf is a county in the Nordwestmecklenb...      B  \n3                 Between 0 and 100 degrees Celsius.      D  \n4  The Saint Lucia Kings won 10 games out of 14 i...      D  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>prompt</th>\n      <th>A</th>\n      <th>B</th>\n      <th>C</th>\n      <th>D</th>\n      <th>E</th>\n      <th>answer</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>What are the applications of confrontation ana...</td>\n      <td>Military strategy and planning</td>\n      <td>Political decision-making</td>\n      <td>Legal disputes resolution</td>\n      <td>Financial analysis</td>\n      <td>All of the above</td>\n      <td>E</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>How did the English dog breeds and those indig...</td>\n      <td>The English dog breeds and those indigenous to...</td>\n      <td>The English dog breeds had no influence on the...</td>\n      <td>The English dog breeds had a major influence o...</td>\n      <td>The English dog breeds primarily influenced th...</td>\n      <td>The English dog breeds and those indigenous to...</td>\n      <td>E</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2</td>\n      <td>What is Metelsdorf?</td>\n      <td>Metelsdorf is a town in the Nordwestmecklenbur...</td>\n      <td>Metelsdorf is a village in the Nordwestmecklen...</td>\n      <td>Metelsdorf is a municipality in the Nordwestme...</td>\n      <td>Metelsdorf is a city in the Nordwestmecklenbur...</td>\n      <td>Metelsdorf is a county in the Nordwestmecklenb...</td>\n      <td>B</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3</td>\n      <td>What is the temperature range at which ice Ic,...</td>\n      <td>Between -240 and -130 kelvins.</td>\n      <td>Between 220 and 240 degrees Fahrenheit.</td>\n      <td>Between -273 and -100 degrees Celsius.</td>\n      <td>Between -33 degrees Celsius and 240 kelvins.</td>\n      <td>Between 0 and 100 degrees Celsius.</td>\n      <td>D</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>4</td>\n      <td>How did the Saint Lucia Kings perform in the f...</td>\n      <td>The Saint Lucia Kings won 6 games out of 14 in...</td>\n      <td>The Saint Lucia Kings won all 14 games in the ...</td>\n      <td>The Saint Lucia Kings did not win any games in...</td>\n      <td>The Saint Lucia Kings won 4 games out of 14 in...</td>\n      <td>The Saint Lucia Kings won 10 games out of 14 i...</td>\n      <td>D</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"data.to_csv(\"./Shuffled Data.csv\", index=False)\n# data = load_dataset(\"csv\", data_files=\"/kaggle/working/Shuffled Data.csv\", split=\"train\")\ndata = load_dataset(\"csv\", data_files=\"/kaggle/input/kaggle-llm-science-exam/train.csv\", split=\"train\")\ndata","metadata":{"execution":{"iopub.status.busy":"2023-08-07T12:19:58.524641Z","iopub.execute_input":"2023-08-07T12:19:58.525046Z","iopub.status.idle":"2023-08-07T12:19:59.050887Z","shell.execute_reply.started":"2023-08-07T12:19:58.525002Z","shell.execute_reply":"2023-08-07T12:19:59.049910Z"},"trusted":true},"execution_count":5,"outputs":[{"name":"stdout","text":"Downloading and preparing dataset csv/default to /root/.cache/huggingface/datasets/csv/default-2760266806ba7114/0.0.0/433e0ccc46f9880962cc2b12065189766fbb2bee57a221866138fb9203c83519...\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Downloading data files:   0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"755e6b2763d84ffd86f865498adf324c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Extracting data files:   0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9815ecc0c99242b18d6b07c904426935"}},"metadata":{}},{"name":"stdout","text":"Dataset csv downloaded and prepared to /root/.cache/huggingface/datasets/csv/default-2760266806ba7114/0.0.0/433e0ccc46f9880962cc2b12065189766fbb2bee57a221866138fb9203c83519. Subsequent calls will reuse this data.\n","output_type":"stream"},{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/datasets/packaged_modules/csv/csv.py:154: FutureWarning: the 'mangle_dupe_cols' keyword is deprecated and will be removed in a future version. Please take steps to stop the use of 'mangle_dupe_cols'\n  csv_file_reader = pd.read_csv(file, iterator=True, dtype=dtype, **self.config.read_csv_kwargs)\n","output_type":"stream"},{"execution_count":5,"output_type":"execute_result","data":{"text/plain":"Dataset({\n    features: ['id', 'prompt', 'A', 'B', 'C', 'D', 'E', 'answer'],\n    num_rows: 8835\n})"},"metadata":{}}]},{"cell_type":"code","source":"template = \"\"\"\nAnswer the following multiple choice question by giving the most appropriate response. Answer should be one among [A, B, C, D, E]\\n\nQuestion: {prompt}\\n\nA) {a}\\n\nB) {b}\\n\nC) {c}\\n\nD) {d}\\n\nE) {e}\\n\nAnswer: {answer}\"\"\"\n\nprompt = PromptTemplate(template=template, input_variables=[\"prompt\", \"a\", \"b\", \"c\", \"d\", \"e\", \"answer\"])","metadata":{"execution":{"iopub.status.busy":"2023-08-07T12:19:59.054429Z","iopub.execute_input":"2023-08-07T12:19:59.054869Z","iopub.status.idle":"2023-08-07T12:19:59.061912Z","shell.execute_reply.started":"2023-08-07T12:19:59.054816Z","shell.execute_reply":"2023-08-07T12:19:59.060844Z"},"trusted":true},"execution_count":6,"outputs":[]},{"cell_type":"code","source":"sample = data[0]\ndisplay(Markdown(prompt.format(prompt=sample[\"prompt\"], a=sample[\"A\"], \n                               b=sample[\"B\"], c=sample[\"C\"], d=sample[\"D\"], \n                               e=sample[\"E\"], answer=sample[\"answer\"])))","metadata":{"execution":{"iopub.status.busy":"2023-08-07T12:19:59.063551Z","iopub.execute_input":"2023-08-07T12:19:59.064106Z","iopub.status.idle":"2023-08-07T12:19:59.078872Z","shell.execute_reply.started":"2023-08-07T12:19:59.064060Z","shell.execute_reply":"2023-08-07T12:19:59.077957Z"},"trusted":true},"execution_count":7,"outputs":[{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.Markdown object>","text/markdown":"\nAnswer the following multiple choice question by giving the most appropriate response. Answer should be one among [A, B, C, D, E]\n\nQuestion: What are the applications of confrontation analysis?\n\nA) Military strategy and planning\n\nB) Political decision-making\n\nC) Legal disputes resolution\n\nD) Financial analysis\n\nE) All of the above\n\nAnswer: E"},"metadata":{}}]},{"cell_type":"code","source":"def format_text(example):\n    text = prompt.format(prompt=example[\"prompt\"], a=example[\"A\"], \n                         b=example[\"B\"], c=example[\"C\"], d=example[\"D\"], \n                         e=example[\"E\"], answer=example[\"answer\"])\n    return {\"text\": text}\n\ndata = data.map(format_text)\ndata","metadata":{"execution":{"iopub.status.busy":"2023-08-07T12:19:59.080506Z","iopub.execute_input":"2023-08-07T12:19:59.081418Z","iopub.status.idle":"2023-08-07T12:20:00.773957Z","shell.execute_reply.started":"2023-08-07T12:19:59.081380Z","shell.execute_reply":"2023-08-07T12:20:00.772871Z"},"trusted":true},"execution_count":8,"outputs":[{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8835 [00:00<?, ?ex/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2f6829534fd84599a37d3bab8b735d17"}},"metadata":{}},{"execution_count":8,"output_type":"execute_result","data":{"text/plain":"Dataset({\n    features: ['id', 'prompt', 'A', 'B', 'C', 'D', 'E', 'answer', 'text'],\n    num_rows: 8835\n})"},"metadata":{}}]},{"cell_type":"code","source":"def plot_sequence_lengths(data, split=\"train\", max_length=2048):\n    sequence_lengths = []\n    selected_indices = []\n    \n    for idx, example in tqdm(enumerate(data), total=len(data)):\n        sequence_lengths.append(len(example[\"text\"]))\n        if sequence_lengths[idx] < max_length:\n            selected_indices.append(idx)\n\n    plt.hist(sequence_lengths, bins=30)\n    plt.xlabel(\"Sequence Length\")\n    plt.ylabel(\"Count\")\n    plt.title(\"Distribution of Text Sequence Lengths\")\n    plt.show()\n    \n    print(\"Max Sequence Length:\", max(sequence_lengths))\n    print(\"Min Sequence Length:\", min(sequence_lengths))\n\n    return selected_indices\n\nkeep_indices_train = plot_sequence_lengths(data)\ndata = data.select(keep_indices_train)\nprint(\"The length of selected data:\", len(data))","metadata":{"execution":{"iopub.status.busy":"2023-08-07T12:20:00.775735Z","iopub.execute_input":"2023-08-07T12:20:00.776133Z","iopub.status.idle":"2023-08-07T12:20:02.584697Z","shell.execute_reply.started":"2023-08-07T12:20:00.776099Z","shell.execute_reply":"2023-08-07T12:20:02.583671Z"},"trusted":true},"execution_count":9,"outputs":[{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8835 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"cafda5a058744a86ab091dd2fe9cf184"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<Figure size 640x480 with 1 Axes>","image/png":"iVBORw0KGgoAAAANSUhEUgAAAkQAAAHFCAYAAAAT5Oa6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAABFwUlEQVR4nO3deVhV5f7//9eWGYQtoIA4gWZOOKVmmoXmmJqZleaUfrKTHkdKS/3qcfokDqfU06Bmp9Qys+GIx9LK2ZNHOhpGDo2ecEqQMgJMRYX794c/18ct4IAoG9fzcV37utz3utda93vvDby817AdxhgjAAAAGytT0gMAAAAoaQQiAABgewQiAABgewQiAABgewQiAABgewQiAABgewQiAABgewQiAABgewQiAABgewQiuJ0lS5bI4XBYD19fX0VERKhNmzaaMWOG0tPT860zZcoUORyOa9rPyZMnNWXKFG3ZsuWa1itoX1FRUerates1bedKli9frnnz5hW4zOFwaMqUKcW6v+K2ceNGNW3aVAEBAXI4HFq1alW+Pq1bt3Z5rwt7FGet8fHxBY6lMMePH9f48eNVt25dBQQEyOl0qnbt2urfv792795dbOOyo9atWysmJqakh1GotWvXFvrZczgcGj58+M0dEG4oz5IeAFCYxYsXq3bt2jp79qzS09O1bds2zZo1Sy+88ILee+89tWvXzur75JNPqlOnTte0/ZMnT2rq1KmSzv9ivlpF2VdRLF++XHv37lVcXFy+ZYmJiapcufINH0NRGWPUs2dP3X777Vq9erUCAgJUq1atfP3mz5+vrKws6/maNWv0/PPPW+/9BcVZa3x8vB555BF17979in1PnDihu+66SydOnNCzzz6rhg0b6tSpU/rhhx+0cuVKJScnq0GDBsU2NriXtWvX6tVXX3X7/3ygeBCI4LZiYmLUtGlT6/nDDz+sp59+Wq1atVKPHj30448/Kjw8XNL5P5g3OiCcPHlS/v7+N2VfV3LXXXeV6P6v5OjRo/rtt9/00EMPqW3btoX2q1u3rsvz7777TlL+976kfPDBB9q/f782bdqkNm3auCx75plnlJeXV0IjA1DcOGSGUqVq1ap68cUXlZ2drddee81qL+gw1qZNm9S6dWuFhobKz89PVatW1cMPP6yTJ0/qwIEDqlChgiRp6tSp1qGZgQMHumxv165deuSRRxQcHKwaNWoUuq8LEhIS1KBBA/n6+qp69ep66aWXXJZfOBx44MABl/YtW7bI4XBYh+9at26tNWvW6ODBgy6Hji4o6DDS3r179eCDDyo4OFi+vr5q1KiRli5dWuB+3n33XU2YMEGRkZEKCgpSu3bt9P333xf+wl9k27Ztatu2rQIDA+Xv76+WLVtqzZo11vIpU6ZYgXHs2LFyOByKioq6qm0X5r333lOLFi0UEBCgsmXLqmPHjvrqq69cxuTl5aUxY8a4rHfh9X7jjTcknX/d/vjjDy1dutR6TS83O3j8+HFJUsWKFQtcXqaM66/QH3/8UX369FFYWJh8fHxUp04dvfrqq/nW++6779SpUyf5+/urfPnyGjJkiD766COXz4B0/lDshc/kxVq3bp1v3FlZWRozZoyio6Pl7e2tSpUqKS4uTn/88YdLvwuHet5++23VqVNH/v7+atiwoT7++OMCx9m7d2+Fh4fLx8dHVatW1eOPP66cnByrT1pamgYPHqzKlSvL29tb0dHRmjp1qs6dO1fga1YUV3r/JWngwIEqW7as9u/fr86dO6ts2bKqUqWKRo8e7TJeSTpy5IgeeeQRBQYGqly5curbt6927twph8OhJUuWWNu78N5d/DN46c/ulV7HX375RU899ZSqVKkiHx8fVahQQXfffbc2bNhQbK8PiokB3MzixYuNJLNz584Cl584ccJ4eHiYtm3bWm2TJ082F3+cU1JSjK+vr2nfvr1ZtWqV2bJli3nnnXdM//79TUZGhjl9+rT59NNPjSQzaNAgk5iYaBITE83+/ftdtletWjUzduxYs379erNq1aoC92WMMdWqVTOVKlUyVatWNW+++aZZu3at6du3r5Fk/vrXv+arLSUlxWX9zZs3G0lm8+bNxhhj9u3bZ+6++24TERFhjS0xMdHqL8lMnjzZev7dd9+ZwMBAU6NGDfPWW2+ZNWvWmN69extJZtasWfn2ExUVZfr27WvWrFlj3n33XVO1alVTs2ZNc+7cucu+N1u2bDFeXl6mSZMm5r333jOrVq0yHTp0MA6Hw6xYscIYY8zhw4fNypUrjSQzYsQIk5iYaHbt2nXZ7V76+lz83k+fPt04HA7zxBNPmI8//tisXLnStGjRwgQEBJh9+/ZZ/WbOnGkkmX/+85/GGGP27t1r/P39Tb9+/aw+iYmJxs/Pz3Tu3Nl6TS/exqW2bdtmJJlmzZqZhIQE8+uvvxbad9++fcbpdJr69eubt956y6xbt86MHj3alClTxkyZMsXql5aWZsLCwkylSpXM4sWLrc9K1apVXT4Dxpz/XA0YMCDfvmJjY01sbKz1/I8//jCNGjUy5cuXN3PmzDEbNmwwf/vb34zT6TT33XefycvLs/peeP/vvPNO8/7775u1a9ea1q1bG09PT/Pf//7X6pecnGzKli1roqKizMKFC83GjRvNsmXLTM+ePU1WVpYxxpjU1FRTpUoVU61aNfPaa6+ZDRs2mP/93/81Pj4+ZuDAgYW+VhfXUa9evcv2udr3f8CAAcbb29vUqVPHvPDCC2bDhg1m0qRJxuFwmKlTp1r9Tpw4YW677TYTEhJiXn31VfPZZ5+Zp59+2kRHRxtJZvHixcYYY/bv328eeeQRI8nlZ/D06dPX9Dp27NjRVKhQwSxatMhs2bLFrFq1ykyaNMn6eYH7IBDB7VwpEBljTHh4uKlTp471/NKQ8uGHHxpJJjk5udBt/PLLL/mCxaXbmzRpUqHLLlatWjXjcDjy7a99+/YmKCjI/PHHHy61XSkQGWNMly5dTLVq1Qoc+6Xjfuyxx4yPj485dOiQS7/777/f+Pv7m99//91lP507d3bp9/7771u/+C/nrrvuMmFhYSY7O9tqO3funImJiTGVK1e2/vCmpKTkC4NX49L3/tChQ8bT09OMGDHCpV92draJiIgwPXv2tNry8vJM586dTbly5czevXtN3bp1Te3atc2JEydc1g0ICCgwZBRm2rRpxtvb20gykkx0dLQZMmSI+frrr136dezY0VSuXNlkZma6tA8fPtz4+vqa3377zRhjzNixYwv9rBQ1EM2YMcOUKVMm38/MhZ+DtWvXWm2STHh4uBVqjDkf0sqUKWNmzJhhtd13332mXLlyJj09vdDXZvDgwaZs2bLm4MGDLu0vvPCCkXTZsHmhjssFomt5/wcMGGAkmffff9+lb+fOnU2tWrWs56+++qqRZD755JN8tVwciIwxZtiwYfl+1i+42texbNmyJi4urtAa4T44ZIZSyRhz2eWNGjWSt7e3nnrqKS1dulQ//fRTkfbz8MMPX3XfevXqqWHDhi5tffr0UVZWlnbt2lWk/V+tTZs2qW3btqpSpYpL+8CBA3Xy5EklJia6tHfr1s3l+YUTgw8ePFjoPv744w/95z//0SOPPKKyZcta7R4eHurfv7+OHDly1YfdrtZnn32mc+fO6fHHH9e5c+esh6+vr2JjY10OLzkcDr311lsKDAxU06ZNlZKSovfff18BAQHXNYa//OUvOnTokN58800NHjxYZcuW1cKFC9WkSRO9++67kqTTp09r48aNeuihh+Tv7+8y1s6dO+v06dP64osvJEmbN28u9LNSVB9//LFiYmLUqFEjl3137Ngx32E4SWrTpo0CAwOt5+Hh4QoLC7Pe/5MnT2rr1q3q2bOndWi5sP22adNGkZGRLvu9//77JUlbt24tck3Stb3/0vnPwAMPPODS1qBBA5fP9datWxUYGJjvwojevXtf8/iu9DpK0p133qklS5bo+eef1xdffKGzZ89e835wcxCIUOr88ccfOn78uCIjIwvtU6NGDW3YsEFhYWEaNmyYatSooRo1auhvf/vbNe2rsHNHChIREVFo24VzUW6U48ePFzjWC6/RpfsPDQ11ee7j4yNJOnXqVKH7yMjIkDHmmvZzvY4dOyZJatasmby8vFwe7733nn799VeX/qGhoerWrZtOnz6tTp06qX79+sUyjvDwcP3P//yPFi5cqN27d2vr1q3y9vbWqFGjJJ2v+9y5c3r55ZfzjbNz586SZI31+PHjl/2sFMWxY8e0e/fufPsODAyUMabA1+lSPj4+1vufkZGh3NzcK148cOzYMX300Uf59luvXj1JyrffotQlXf377+/vL19f33x1nT592np+/Phx62KMixXUdiVXeh2l8+c/DRgwQH//+9/VokULhYSE6PHHH1daWto17w83FleZodRZs2aNcnNzr3ip/D333KN77rlHubm5+vLLL/Xyyy8rLi5O4eHheuyxx65qX9dyb6OCfsFdaLvwi/PCL+tLT/K83j8coaGhSk1Nzdd+9OhRSVL58uWva/uSFBwcrDJlytzw/VzswvY+/PBDVatW7Yr9169frwULFujOO+9UQkKC/vGPf1zTLN/Vuvfee9WhQwetWrVK6enpCg4OtmbKhg0bVuA60dHRks6/V5f7rFzM19c332dFOv95ufi1Ll++vPz8/PTmm28WuO9rfV9CQkLk4eGhI0eOXLZf+fLl1aBBA02fPr3A5Zf7T8vVuNb3/2qEhoZqx44d+dpvVEApX7685s2bp3nz5unQoUNavXq1xo0bp/T0dH366ac3ZJ8oGgIRSpVDhw5pzJgxcjqdGjx48FWt4+HhoebNm6t27dp65513tGvXLj322GNXNStyLfbt26evv/7a5VDI8uXLFRgYqDvuuEOSrKutdu/e7XJfntWrV+fb3qX/07yctm3bKiEhQUePHnX5I/TWW2/J39+/WC7TDwgIUPPmzbVy5Uq98MIL8vPzkyTl5eVp2bJlqly5sm6//fbr3s/FOnbsKE9PT/33v/+9YrBJTU1Vv379FBsbq/Xr16tHjx4aNGiQ7rjjDiuMSNf2uh47dkwVKlTIdzVZbm6ufvzxR/n7+6tcuXLy9vZWmzZt9NVXX6lBgwby9vYudJtt2rTR7NmzC/ysXCoqKirfzR9/+OEHff/99y4hp2vXroqPj1doaKhLrUXl5+en2NhYffDBB5o+fXqhgapr165au3atatSooeDg4Ove76Wu5f2/WrGxsXr//ff1ySefWIf2JGnFihX5+l78O+LC5/16VK1aVcOHD9fGjRv173//+7q3h+JFIILb2rt3r3XOQHp6uj7//HMtXrxYHh4eSkhIuOy5DQsXLtSmTZvUpUsXVa1aVadPn7b+93zhho6BgYGqVq2a/vnPf6pt27YKCQlR+fLli3yJeGRkpLp166YpU6aoYsWKWrZsmdavX69Zs2bJ399f0vmp/1q1amnMmDE6d+6cgoODlZCQoG3btuXbXv369bVy5UotWLBATZo0UZkyZQq9N8/kyZOt8zkmTZqkkJAQvfPOO1qzZo1mz54tp9NZpJouNWPGDLVv315t2rTRmDFj5O3trfnz52vv3r169913r/lu4VcSFRWladOmacKECfrpp5/UqVMnBQcH69ixY9qxY4cCAgI0depU5ebmqnfv3nI4HFq+fLk8PDy0ZMkSNWrUSL169dK2bduskFK/fn1t2bJFH330kSpWrKjAwMACbxopnb+k+rXXXlOfPn3UrFkzOZ1OHTlyRH//+9+1b98+TZo0ydru3/72N7Vq1Ur33HOP/vznPysqKkrZ2dnav3+/PvroI23atEmSFBcXpzfffFNdunTR888/r/DwcL3zzjvWPZgu1r9/f/Xr109Dhw7Vww8/rIMHD2r27Nn5PvtxcXH6xz/+oXvvvVdPP/20GjRooLy8PB06dEjr1q3T6NGj1bx582t67efMmaNWrVqpefPmGjdunG677TYdO3ZMq1ev1muvvabAwEBNmzZN69evV8uWLTVy5EjVqlVLp0+f1oEDB7R27VotXLjwiofdsrKy9OGHH+Zrr1ChgmJjY6/q/b8WAwYM0Ny5c9WvXz89//zzuu222/TJJ5/os88+k+R6K4ULh1xnzZql+++/Xx4eHlcMvBfLzMxUmzZt1KdPH9WuXVuBgYHauXOnPv30U/Xo0eOaxo2boIRP6gbyuXCl0YWHt7e3CQsLM7GxsSY+Pr7Aq14uvfIrMTHRPPTQQ6ZatWrGx8fHhIaGmtjYWLN69WqX9TZs2GAaN25sfHx8jCTrip4L2/vll1+uuC9jzl8N1KVLF/Phhx+aevXqGW9vbxMVFWXmzJmTb/0ffvjBdOjQwQQFBZkKFSqYESNGmDVr1uS7wui3334zjzzyiClXrpxxOBwu+1QBV8ft2bPHPPDAA8bpdBpvb2/TsGFDlytmjPm/q8w++OADl/YLV4Vd2r8gn3/+ubnvvvtMQECA8fPzM3fddZf56KOPCtze9V5ldsGqVatMmzZtTFBQkPHx8THVqlUzjzzyiNmwYYMxxpgJEyaYMmXKmI0bN7qst337duPp6WlGjRpltSUnJ5u7777b+Pv7G0kuV2td6ptvvjGjR482TZs2NRUqVDCenp4mODjYxMbGmrfffjtf/5SUFPPEE0+YSpUqGS8vL1OhQgXTsmVL8/zzz+fbbvv27Y2vr68JCQkxgwYNMv/85z/zfQby8vLM7NmzTfXq1Y2vr69p2rSp2bRpU76rzIw5fzn5xIkTTa1atYy3t7d1C4Cnn37apKWlWf0kmWHDhuUbe0FXtH3zzTfm0UcfNaGhocbb29tUrVrVDBw40Lr03JjzV2uOHDnSREdHGy8vLxMSEmKaNGliJkyYkO8Kv0vFxsa6/Kxf/Li4viu9/8acv8osICAg3z4K+nk9dOiQ6dGjhylbtqwJDAw0Dz/8sFm7dq3LbRuMMSYnJ8c8+eSTpkKFCtbP4IUrRK/mdTx9+rQZMmSIadCggQkKCjJ+fn6mVq1aZvLkydaVp3AfDmOucLkOAOCG27Jli9q0aaPNmzdf01fJoHjEx8dr4sSJOnToUInfiR4lg0NmAABbeeWVVyTJ+q7ETZs26aWXXlK/fv0IQzZGIAIA2Iq/v7/mzp2rAwcOKCcnR1WrVtXYsWM1ceLEkh4aShCHzAAAgO1xY0YAAGB7BCIAAGB7BCIAAGB7nFR9lfLy8nT06FEFBgYW+83nAADAjWGMUXZ2tiIjI/Pddf5iBKKrdPTo0XzfJA4AAEqHw4cPX/a2CgSiqxQYGCjp/AsaFBRUwqMBAABXIysrS1WqVLH+jhemRAPRv/71L/31r39VUlKSUlNTlZCQoO7du1vLjTGaOnWqFi1apIyMDDVv3lyvvvqq6tWrZ/XJycnRmDFj9O677+rUqVNq27at5s+f75ICMzIyNHLkSOsLNLt166aXX35Z5cqVu+qxXjhMFhQURCACAKCUudLpLiV6UvUff/yhhg0bWncNvdTs2bM1Z84cvfLKK9q5c6ciIiLUvn17ZWdnW33i4uKUkJCgFStWaNu2bTpx4oS6du2q3Nxcq0+fPn2UnJysTz/9VJ9++qmSk5PVv3//G14fAAAoJUryi9QuJskkJCRYz/Py8kxERISZOXOm1Xb69GnjdDrNwoULjTHG/P7778bLy8usWLHC6vPzzz+bMmXKmE8//dQYc/7LCSWZL774wuqTmJhoJJnvvvvuqseXmZlpJJnMzMyilggAAG6yq/377baX3aekpCgtLU0dOnSw2nx8fBQbG6vt27dLkpKSknT27FmXPpGRkYqJibH6JCYmyul0qnnz5lafu+66S06n0+pTkJycHGVlZbk8AADArcltA1FaWpokKTw83KU9PDzcWpaWliZvb28FBwdftk9YWFi+7YeFhVl9CjJjxgw5nU7rwRVmAADcutw2EF1w6UlQxpgrnhh1aZ+C+l9pO+PHj1dmZqb1OHz48DWOHAAAlBZuG4giIiIkKd8sTnp6ujVrFBERoTNnzigjI+OyfY4dO5Zv+7/88ku+2aeL+fj4WFeUcWUZAAC3NrcNRNHR0YqIiND69euttjNnzmjr1q1q2bKlJKlJkyby8vJy6ZOamqq9e/dafVq0aKHMzEzt2LHD6vOf//xHmZmZVh8AAGBvJXofohMnTmj//v3W85SUFCUnJyskJERVq1ZVXFyc4uPjVbNmTdWsWVPx8fHy9/dXnz59JElOp1ODBg3S6NGjFRoaqpCQEI0ZM0b169dXu3btJEl16tRRp06d9Kc//UmvvfaaJOmpp55S165dVatWrZtfNAAAcDslGoi+/PJLtWnTxnr+zDPPSJIGDBigJUuW6LnnntOpU6c0dOhQ68aM69atc7nb5Ny5c+Xp6amePXtaN2ZcsmSJPDw8rD7vvPOORo4caV2N1q1bt0LvfQQAAOzHYYwxJT2I0iArK0tOp1OZmZmcTwQAQClxtX+/3fYcIgAAgJuFQAQAAGyPQAQAAGyPQAQAAGyPQAQAAGyvRC+7x/WLGremyOsemNmlGEcCAEDpxQwRAACwPQIRAACwPQIRAACwPQIRAACwPQIRAACwPQIRAACwPQIRAACwPQIRAACwPQIRAACwPQIRAACwPQIRAACwPQIRAACwPQIRAACwPQIRAACwPQIRAACwPQIRAACwPQIRAACwPQIRAACwPQIRAACwPc+SHgBKTtS4NUVe98DMLsU4EgAAShYzRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPbcOhCdO3dOEydOVHR0tPz8/FS9enVNmzZNeXl5Vh9jjKZMmaLIyEj5+fmpdevW2rdvn8t2cnJyNGLECJUvX14BAQHq1q2bjhw5crPLAQAAbsqtA9GsWbO0cOFCvfLKK/r22281e/Zs/fWvf9XLL79s9Zk9e7bmzJmjV155RTt37lRERITat2+v7Oxsq09cXJwSEhK0YsUKbdu2TSdOnFDXrl2Vm5tbEmUBAAA341nSA7icxMREPfjgg+rSpYskKSoqSu+++66+/PJLSednh+bNm6cJEyaoR48ekqSlS5cqPDxcy5cv1+DBg5WZmak33nhDb7/9ttq1aydJWrZsmapUqaINGzaoY8eOJVMcAABwG249Q9SqVStt3LhRP/zwgyTp66+/1rZt29S5c2dJUkpKitLS0tShQwdrHR8fH8XGxmr79u2SpKSkJJ09e9alT2RkpGJiYqw+BcnJyVFWVpbLAwAA3JrceoZo7NixyszMVO3ateXh4aHc3FxNnz5dvXv3liSlpaVJksLDw13WCw8P18GDB60+3t7eCg4OztfnwvoFmTFjhqZOnVqc5QAAADfl1jNE7733npYtW6bly5dr165dWrp0qV544QUtXbrUpZ/D4XB5bozJ13apK/UZP368MjMzrcfhw4eLXggAAHBrbj1D9Oyzz2rcuHF67LHHJEn169fXwYMHNWPGDA0YMEARERGSzs8CVaxY0VovPT3dmjWKiIjQmTNnlJGR4TJLlJ6erpYtWxa6bx8fH/n4+NyIsgAAgJtx6xmikydPqkwZ1yF6eHhYl91HR0crIiJC69evt5afOXNGW7dutcJOkyZN5OXl5dInNTVVe/fuvWwgAgAA9uHWM0QPPPCApk+frqpVq6pevXr66quvNGfOHD3xxBOSzh8qi4uLU3x8vGrWrKmaNWsqPj5e/v7+6tOnjyTJ6XRq0KBBGj16tEJDQxUSEqIxY8aofv361lVnAADA3tw6EL388sv6y1/+oqFDhyo9PV2RkZEaPHiwJk2aZPV57rnndOrUKQ0dOlQZGRlq3ry51q1bp8DAQKvP3Llz5enpqZ49e+rUqVNq27atlixZIg8Pj5IoCwAAuBmHMcaU9CBKg6ysLDmdTmVmZiooKKikh2OJGremRPZ7YGaXEtkvAADX4mr/frv1OUQAAAA3A4EIAADYHoEIAADYHoEIAADYnltfZQa4k+s5gZ2T0AHAvTFDBAAAbI9ABAAAbI9ABAAAbI9ziGArJXUjSwCAe2OGCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2J5nSQ8AsIOocWuKvO6BmV2KcSQAgIIwQwQAAGyPQAQAAGyPQAQAAGyPQAQAAGyPQAQAAGyPQAQAAGyPQAQAAGyPQAQAAGyPQAQAAGyPO1WjVLmeOz4DAFAYZogAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDt8eWuKJLr+ZLVAzO7FONIAAC4fswQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA23P7QPTzzz+rX79+Cg0Nlb+/vxo1aqSkpCRruTFGU6ZMUWRkpPz8/NS6dWvt27fPZRs5OTkaMWKEypcvr4CAAHXr1k1Hjhy52aUAAAA35daBKCMjQ3fffbe8vLz0ySef6JtvvtGLL76ocuXKWX1mz56tOXPm6JVXXtHOnTsVERGh9u3bKzs72+oTFxenhIQErVixQtu2bdOJEyfUtWtX5ebmlkBVAADA3bj1jRlnzZqlKlWqaPHixVZbVFSU9W9jjObNm6cJEyaoR48ekqSlS5cqPDxcy5cv1+DBg5WZmak33nhDb7/9ttq1aydJWrZsmapUqaINGzaoY8eON7UmAADgftx6hmj16tVq2rSpHn30UYWFhalx48Z6/fXXreUpKSlKS0tThw4drDYfHx/FxsZq+/btkqSkpCSdPXvWpU9kZKRiYmKsPgAAwN7cOhD99NNPWrBggWrWrKnPPvtMQ4YM0ciRI/XWW29JktLS0iRJ4eHhLuuFh4dby9LS0uTt7a3g4OBC+xQkJydHWVlZLg8AAHBrcutDZnl5eWratKni4+MlSY0bN9a+ffu0YMECPf7441Y/h8Phsp4xJl/bpa7UZ8aMGZo6dep1jB6FuZ7vQQMA4EZw6xmiihUrqm7dui5tderU0aFDhyRJERERkpRvpic9Pd2aNYqIiNCZM2eUkZFRaJ+CjB8/XpmZmdbj8OHD110PAABwT24diO6++259//33Lm0//PCDqlWrJkmKjo5WRESE1q9fby0/c+aMtm7dqpYtW0qSmjRpIi8vL5c+qamp2rt3r9WnID4+PgoKCnJ5AACAW5NbHzJ7+umn1bJlS8XHx6tnz57asWOHFi1apEWLFkk6f6gsLi5O8fHxqlmzpmrWrKn4+Hj5+/urT58+kiSn06lBgwZp9OjRCg0NVUhIiMaMGaP69etbV50BAAB7c+tA1KxZMyUkJGj8+PGaNm2aoqOjNW/ePPXt29fq89xzz+nUqVMaOnSoMjIy1Lx5c61bt06BgYFWn7lz58rT01M9e/bUqVOn1LZtWy1ZskQeHh4lURZwTa7nnKsDM7sU40gA4NblMMaYkh5EaZCVlSWn06nMzEy3OnzGCcq4HAIRALu72r/fbn0OEQAAwM1AIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZXpEBUvXp1HT9+PF/777//rurVq1/3oAAAAG6mIgWiAwcOKDc3N197Tk6Ofv755+seFAAAwM10TV/uunr1auvfn332mZxOp/U8NzdXGzduVFRUVLENDgAA4Ga4pkDUvXt3SZLD4dCAAQNclnl5eSkqKkovvvhisQ0OAADgZrimQJSXlydJio6O1s6dO1W+fPkbMigAAICb6ZoC0QUpKSnFPQ4AAIASU6RAJEkbN27Uxo0blZ6ebs0cXfDmm29e98AAAABuliIFoqlTp2ratGlq2rSpKlasKIfDUdzjAgAAuGmKFIgWLlyoJUuWqH///sU9HgAAgJuuSPchOnPmjFq2bFncYwEAACgRRQpETz75pJYvX17cYwEAACgRRTpkdvr0aS1atEgbNmxQgwYN5OXl5bJ8zpw5xTI4AACAm6FIgWj37t1q1KiRJGnv3r0uyzjBGgAAlDZFCkSbN28u7nEAAACUmCKdQwQAAHArKdIMUZs2bS57aGzTpk1FHhAAAMDNVqRAdOH8oQvOnj2r5ORk7d27N9+XvgIAALi7IgWiuXPnFtg+ZcoUnThx4roGBAAAcLMV6zlE/fr143vMAABAqVOsgSgxMVG+vr7FuUkAAIAbrkiHzHr06OHy3Bij1NRUffnll/rLX/5SLAMDAAC4WYoUiJxOp8vzMmXKqFatWpo2bZo6dOhQLAMDAAC4WYoUiBYvXlzc4wAAACgxRQpEFyQlJenbb7+Vw+FQ3bp11bhx4+IaFwAAwE1TpECUnp6uxx57TFu2bFG5cuVkjFFmZqbatGmjFStWqEKFCsU9TgAAgBumSFeZjRgxQllZWdq3b59+++03ZWRkaO/evcrKytLIkSOLe4wAAAA3lMMYY651JafTqQ0bNqhZs2Yu7Tt27FCHDh30+++/F9f43EZWVpacTqcyMzMVFBRU0sOxRI1bU9JDwC3qwMwuJT0EALhuV/v3u0gzRHl5efLy8srX7uXlpby8vKJsEgAAoMQUKRDdd999GjVqlI4ePWq1/fzzz3r66afVtm3bYhscAADAzVCkQPTKK68oOztbUVFRqlGjhm677TZFR0crOztbL7/8cnGPEQAA4IYq0lVmVapU0a5du7R+/Xp99913Msaobt26ateuXXGPDwAA4Ia7phmiTZs2qW7dusrKypIktW/fXiNGjNDIkSPVrFkz1atXT59//vkNGSgAAMCNck2BaN68efrTn/5U4FnaTqdTgwcP1pw5c4ptcAAAADfDNQWir7/+Wp06dSp0eYcOHZSUlHTdgwIAALiZrikQHTt2rMDL7S/w9PTUL7/8ct2DAgAAuJmuKRBVqlRJe/bsKXT57t27VbFixeseFAAAwM10TYGoc+fOmjRpkk6fPp1v2alTpzR58mR17dq12AYHAABwM1zTZfcTJ07UypUrdfvtt2v48OGqVauWHA6Hvv32W7366qvKzc3VhAkTbtRYAQAAbohrCkTh4eHavn27/vznP2v8+PG68DVoDodDHTt21Pz58xUeHn5DBgoAAHCjXPONGatVq6a1a9cqIyND+/fvlzFGNWvWVHBw8I0YHwAAwA1XpDtVS1JwcHC+b7sHAAAojYr0XWYAAAC3EgIRAACwPQIRAACwPQIRAACwPQIRAACwPQIRAACwPQIRAACwPQIRAACwPQIRAACwPQIRAACwPQIRAACwPQIRAACwvVIViGbMmCGHw6G4uDirzRijKVOmKDIyUn5+fmrdurX27dvnsl5OTo5GjBih8uXLKyAgQN26ddORI0du8ugBAIC7KjWBaOfOnVq0aJEaNGjg0j579mzNmTNHr7zyinbu3KmIiAi1b99e2dnZVp+4uDglJCRoxYoV2rZtm06cOKGuXbsqNzf3ZpcBAADcUKkIRCdOnFDfvn31+uuvKzg42Go3xmjevHmaMGGCevTooZiYGC1dulQnT57U8uXLJUmZmZl644039OKLL6pdu3Zq3Lixli1bpj179mjDhg0lVRIAAHAjpSIQDRs2TF26dFG7du1c2lNSUpSWlqYOHTpYbT4+PoqNjdX27dslSUlJSTp79qxLn8jISMXExFh9CpKTk6OsrCyXBwAAuDV5lvQArmTFihXatWuXdu7cmW9ZWlqaJCk8PNylPTw8XAcPHrT6eHt7u8wsXehzYf2CzJgxQ1OnTr3e4QMAgFLArWeIDh8+rFGjRmnZsmXy9fUttJ/D4XB5bozJ13apK/UZP368MjMzrcfhw4evbfAAAKDUcOtAlJSUpPT0dDVp0kSenp7y9PTU1q1b9dJLL8nT09OaGbp0pic9Pd1aFhERoTNnzigjI6PQPgXx8fFRUFCQywMAANya3DoQtW3bVnv27FFycrL1aNq0qfr27avk5GRVr15dERERWr9+vbXOmTNntHXrVrVs2VKS1KRJE3l5ebn0SU1N1d69e60+AADA3tz6HKLAwEDFxMS4tAUEBCg0NNRqj4uLU3x8vGrWrKmaNWsqPj5e/v7+6tOnjyTJ6XRq0KBBGj16tEJDQxUSEqIxY8aofv36+U7SBgAA9uTWgehqPPfcczp16pSGDh2qjIwMNW/eXOvWrVNgYKDVZ+7cufL09FTPnj116tQptW3bVkuWLJGHh0cJjhwAALgLhzHGlPQgSoOsrCw5nU5lZma61flEUePWlPQQcIs6MLNLSQ8BAK7b1f79dutziAAAAG4GAhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALC9Un8fIgA3xvXc0oFL9gGUNswQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2yMQAQAA2/Ms6QEAuPVEjVtT5HUPzOxSjCMBgKvDDBEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9bszoBq7nJnYAAOD6MUMEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsz7OkBwAAF4sat6bI6x6Y2aUYRwLATpghAgAAtufWgWjGjBlq1qyZAgMDFRYWpu7du+v777936WOM0ZQpUxQZGSk/Pz+1bt1a+/btc+mTk5OjESNGqHz58goICFC3bt105MiRm1kKAABwY24diLZu3aphw4bpiy++0Pr163Xu3Dl16NBBf/zxh9Vn9uzZmjNnjl555RXt3LlTERERat++vbKzs60+cXFxSkhI0IoVK7Rt2zadOHFCXbt2VW5ubkmUBQAA3IzDGGNKehBX65dfflFYWJi2bt2qe++9V8YYRUZGKi4uTmPHjpV0fjYoPDxcs2bN0uDBg5WZmakKFSro7bffVq9evSRJR48eVZUqVbR27Vp17NjxqvadlZUlp9OpzMxMBQUFFWtd13POBID/wzlEAC51tX+/3XqG6FKZmZmSpJCQEElSSkqK0tLS1KFDB6uPj4+PYmNjtX37dklSUlKSzp4969InMjJSMTExVp+C5OTkKCsry+UBAABuTaUmEBlj9Mwzz6hVq1aKiYmRJKWlpUmSwsPDXfqGh4dby9LS0uTt7a3g4OBC+xRkxowZcjqd1qNKlSrFWQ4AAHAjpSYQDR8+XLt379a7776bb5nD4XB5bozJ13apK/UZP368MjMzrcfhw4eLNnAAAOD2SkUgGjFihFavXq3NmzercuXKVntERIQk5ZvpSU9Pt2aNIiIidObMGWVkZBTapyA+Pj4KCgpyeQAAgFuTWwciY4yGDx+ulStXatOmTYqOjnZZHh0drYiICK1fv95qO3PmjLZu3aqWLVtKkpo0aSIvLy+XPqmpqdq7d6/VBwAA2Jtb36l62LBhWr58uf75z38qMDDQmglyOp3y8/OTw+FQXFyc4uPjVbNmTdWsWVPx8fHy9/dXnz59rL6DBg3S6NGjFRoaqpCQEI0ZM0b169dXu3btSrI8AADgJtw6EC1YsECS1Lp1a5f2xYsXa+DAgZKk5557TqdOndLQoUOVkZGh5s2ba926dQoMDLT6z507V56enurZs6dOnTqltm3basmSJfLw8LhZpQAAADdWqu5DVJK4DxHg/rgPEYBL3ZL3IQIAALgRCEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2PEt6AABQXKLGrSnyugdmdinGkQAobZghAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtseXuwKA+GJYwO6YIQIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALbHfYgA4DpxDyOg9GOGCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B5f3QEAJeh6vvbjevCVIYArZogAAIDtEYgAAIDtEYgAAIDtcQ4RANjQ9Zy7xPlHuBURiAAA14QwhVsRh8wAAIDtEYgAAIDtccgMAFAqcKgONxIzRAAAwPaYIQIA3DQldWdu4EqYIQIAALbHDBEA4JbH+Ue4ElsFovnz5+uvf/2rUlNTVa9ePc2bN0/33HNPSQ8LAHCLIoiVHrYJRO+9957i4uI0f/583X333Xrttdd0//3365tvvlHVqlVLengAADfFeU/2YJtziObMmaNBgwbpySefVJ06dTRv3jxVqVJFCxYsKOmhAQCAEmaLGaIzZ84oKSlJ48aNc2nv0KGDtm/fXkKjAgCgcCU1M3U9h+pK8yFCWwSiX3/9Vbm5uQoPD3dpDw8PV1paWoHr5OTkKCcnx3qemZkpScrKyir28eXlnCz2bQIAUBRVn/6gRPZ7I/6+XrxdY8xl+9kiEF3gcDhcnhtj8rVdMGPGDE2dOjVfe5UqVW7I2AAAsDPnvBu7/ezsbDmdzkKX2yIQlS9fXh4eHvlmg9LT0/PNGl0wfvx4PfPMM9bzvLw8/fbbbwoNDS00RF1JVlaWqlSposOHDysoKKhI23BX1FY63cq1Sbd2fdRWet3K9bljbcYYZWdnKzIy8rL9bBGIvL291aRJE61fv14PPfSQ1b5+/Xo9+OCDBa7j4+MjHx8fl7Zy5coVy3iCgoLc5oNS3KitdLqVa5Nu7fqorfS6letzt9ouNzN0gS0CkSQ988wz6t+/v5o2baoWLVpo0aJFOnTokIYMGVLSQwMAACXMNoGoV69eOn78uKZNm6bU1FTFxMRo7dq1qlatWkkPDQAAlDDbBCJJGjp0qIYOHVpi+/fx8dHkyZPzHYq7FVBb6XQr1ybd2vVRW+l1K9dXmmtzmCtdhwYAAHCLs82dqgEAAApDIAIAALZHIAIAALZHIAIAALZHILoOM2bMULNmzRQYGKiwsDB1795d33//vUsfY4ymTJmiyMhI+fn5qXXr1tq3b59Ln5ycHI0YMULly5dXQECAunXrpiNHjtzMUq5oxowZcjgciouLs9pKe20///yz+vXrp9DQUPn7+6tRo0ZKSkqylpfW+s6dO6eJEycqOjpafn5+ql69uqZNm6a8vDyrT2mq7V//+pceeOABRUZGyuFwaNWqVS7Li6uWjIwM9e/fX06nU06nU/3799fvv/9eYrWdPXtWY8eOVf369RUQEKDIyEg9/vjjOnr0aKmv7VKDBw+Ww+HQvHnzXNrdtTbp6ur79ttv1a1bNzmdTgUGBuquu+7SoUOHrOXuWt+Vajtx4oSGDx+uypUry8/PT3Xq1NGCBQtc+rhrbZdlUGQdO3Y0ixcvNnv37jXJycmmS5cupmrVqubEiRNWn5kzZ5rAwEDzj3/8w+zZs8f06tXLVKxY0WRlZVl9hgwZYipVqmTWr19vdu3aZdq0aWMaNmxozp07VxJl5bNjxw4TFRVlGjRoYEaNGmW1l+bafvvtN1OtWjUzcOBA85///MekpKSYDRs2mP3791t9Smt9zz//vAkNDTUff/yxSUlJMR988IEpW7asmTdvntWnNNW2du1aM2HCBPOPf/zDSDIJCQkuy4urlk6dOpmYmBizfft2s337dhMTE2O6du1aYrX9/vvvpl27dua9994z3333nUlMTDTNmzc3TZo0cdlGaaztYgkJCaZhw4YmMjLSzJ0712WZu9ZmzJXr279/vwkJCTHPPvus2bVrl/nvf/9rPv74Y3Ps2DG3r+9KtT355JOmRo0aZvPmzSYlJcW89tprxsPDw6xatcrta7scAlExSk9PN5LM1q1bjTHG5OXlmYiICDNz5kyrz+nTp43T6TQLFy40xpz/pefl5WVWrFhh9fn5559NmTJlzKeffnpzCyhAdna2qVmzplm/fr2JjY21AlFpr23s2LGmVatWhS4vzfV16dLFPPHEEy5tPXr0MP369TPGlO7aLv3lXFy1fPPNN0aS+eKLL6w+iYmJRpL57rvvbnBV510uNFywY8cOI8kcPHjQGFP6azty5IipVKmS2bt3r6lWrZpLICottRlTcH29evWyfuYKUlrqK6i2evXqmWnTprm03XHHHWbixInGmNJT26U4ZFaMMjMzJUkhISGSpJSUFKWlpalDhw5WHx8fH8XGxmr79u2SpKSkJJ09e9alT2RkpGJiYqw+JWnYsGHq0qWL2rVr59Je2mtbvXq1mjZtqkcffVRhYWFq3LixXn/9dWt5aa6vVatW2rhxo3744QdJ0tdff61t27apc+fOkkp3bZcqrloSExPldDrVvHlzq89dd90lp9PpVvVmZmbK4XBY36tYmmvLy8tT//799eyzz6pevXr5lpf22tasWaPbb79dHTt2VFhYmJo3b+5y6Kk019eqVSutXr1aP//8s4wx2rx5s3744Qd17NhRUumtjUBUTIwxeuaZZ9SqVSvFxMRIktLS0iRJ4eHhLn3Dw8OtZWlpafL29lZwcHChfUrKihUrtGvXLs2YMSPfstJe208//aQFCxaoZs2a+uyzzzRkyBCNHDlSb731lqTSXd/YsWPVu3dv1a5dW15eXmrcuLHi4uLUu3dvSaW7tksVVy1paWkKCwvLt/2wsDC3qff06dMaN26c+vTpY31pZmmubdasWfL09NTIkSMLXF6aa0tPT9eJEyc0c+ZMderUSevWrdNDDz2kHj16aOvWrZJKd30vvfSS6tatq8qVK8vb21udOnXS/Pnz1apVK0mltzZbfXXHjTR8+HDt3r1b27Zty7fM4XC4PDfG5Gu71NX0uZEOHz6sUaNGad26dfL19S20X2msTTr/P7imTZsqPj5ektS4cWPt27dPCxYs0OOPP271K431vffee1q2bJmWL1+uevXqKTk5WXFxcYqMjNSAAQOsfqWxtsIURy0F9XeXes+ePavHHntMeXl5mj9//hX7u3ttSUlJ+tvf/qZdu3Zd8xjcvTZJ1gUMDz74oJ5++mlJUqNGjbR9+3YtXLhQsbGxha5bGup76aWX9MUXX2j16tWqVq2a/vWvf2no0KGqWLFivqMJF3P32pghKgYjRozQ6tWrtXnzZlWuXNlqj4iIkKR8aTc9Pd36H21ERITOnDmjjIyMQvuUhKSkJKWnp6tJkyby9PSUp6entm7dqpdeekmenp7W2EpjbZJUsWJF1a1b16WtTp061hUgpfm9e/bZZzVu3Dg99thjql+/vvr376+nn37amukrzbVdqrhqiYiI0LFjx/Jt/5dffinxes+ePauePXsqJSVF69evt2aHpNJb2+eff6709HRVrVrV+v1y8OBBjR49WlFRUZJKb22SVL58eXl6el7xd0xprO/UqVP6f//v/2nOnDl64IEH1KBBAw0fPly9evXSCy+8IKn01kYgug7GGA0fPlwrV67Upk2bFB0d7bI8OjpaERERWr9+vdV25swZbd26VS1btpQkNWnSRF5eXi59UlNTtXfvXqtPSWjbtq327Nmj5ORk69G0aVP17dtXycnJql69eqmtTZLuvvvufLdI+OGHH1StWjVJpfu9O3nypMqUcf3R9vDwsP7XWppru1Rx1dKiRQtlZmZqx44dVp///Oc/yszMLNF6L4ShH3/8URs2bFBoaKjL8tJaW//+/bV7926X3y+RkZF69tln9dlnn0kqvbVJkre3t5o1a3bZ3zGltb6zZ8/q7Nmzl/0dU1pr4yqz6/DnP//ZOJ1Os2XLFpOammo9Tp48afWZOXOmcTqdZuXKlWbPnj2md+/eBV4SXLlyZbNhwwaza9cuc99995X4pdsFufgqM2NKd207duwwnp6eZvr06ebHH38077zzjvH39zfLli2z+pTW+gYMGGAqVapkXXa/cuVKU758efPcc89ZfUpTbdnZ2earr74yX331lZFk5syZY7766ivrSqviqqVTp06mQYMGJjEx0SQmJpr69evf8EuAL1fb2bNnTbdu3UzlypVNcnKyy++YnJycUl1bQS69yswY963NmCvXt3LlSuPl5WUWLVpkfvzxR/Pyyy8bDw8P8/nnn7t9fVeqLTY21tSrV89s3rzZ/PTTT2bx4sXG19fXzJ8/3+1ruxwC0XWQVOBj8eLFVp+8vDwzefJkExERYXx8fMy9995r9uzZ47KdU6dOmeHDh5uQkBDj5+dnunbtag4dOnSTq7mySwNRaa/to48+MjExMcbHx8fUrl3bLFq0yGV5aa0vKyvLjBo1ylStWtX4+vqa6tWrmwkTJrj8ES1NtW3evLnAn7MBAwYUay3Hjx83ffv2NYGBgSYwMND07dvXZGRklFhtKSkphf6O2bx5c6murSAFBSJ3rc2Yq6vvjTfeMLfddpvx9fU1DRs2dLlPjzvXd6XaUlNTzcCBA01kZKTx9fU1tWrVMi+++KLJy8tz+9oux2GMMTdq9gkAAKA04BwiAABgewQiAABgewQiAABgewQiAABgewQiAABgewQiAABgewQiAABgewQiALhFORwOrVq1qqSHAZQKBCIAhUpPT9fgwYNVtWpV+fj4KCIiQh07dlRiYmJJD81tuEPomDJliho1alSiYwBKO8+SHgAA9/Xwww/r7NmzWrp0qapXr65jx45p48aN+u2330p6aABQrJghAlCg33//Xdu2bdOsWbPUpk0bVatWTXfeeafGjx+vLl26WP0yMzP11FNPKSwsTEFBQbrvvvv09ddfu2xr5syZCg8PV2BgoAYNGqRx48a5zGi0bt1acXFxLut0795dAwcOtJ6fOXNGzz33nCpVqqSAgAA1b95cW7ZssZYvWbJE5cqV02effaY6deqobNmy6tSpk1JTU122++abb6pevXry8fFRxYoVNXz48Guq5VotXrxYderUka+vr2rXrq358+dbyw4cOCCHw6GVK1eqTZs28vf3V8OGDfPNwL3++uuqUqWK/P399dBDD2nOnDkqV66cVffUqVP19ddfy+FwyOFwaMmSJda6v/76qx566CH5+/urZs2aWr169XXVA9yqCEQAClS2bFmVLVtWq1atUk5OToF9jDHq0qWL0tLStHbtWiUlJemOO+5Q27ZtrVmk999/X5MnT9b06dP15ZdfqmLFii6h4Gr9z//8j/79739rxYoV2r17tx599FF16tRJP/74o9Xn5MmTeuGFF/T222/rX//6lw4dOqQxY8ZYyxcsWKBhw4bpqaee0p49e7R69WrddtttV13LtXr99dc1YcIETZ8+Xd9++63i4+P1l7/8RUuXLnXpN2HCBI0ZM0bJycm6/fbb1bt3b507d06S9O9//1tDhgzRqFGjlJycrPbt22v69OnWur169dLo0aNVr149paamKjU1Vb169bKWT506VT179tTu3bvVuXNn9e3blxk+oCAl9rWyANzehx9+aIKDg42vr69p2bKlGT9+vPn666+t5Rs3bjRBQUHm9OnTLuvVqFHDvPbaa8YYY1q0aGGGDBnisrx58+amYcOG1vPY2FgzatQolz4PPvig9e3a+/fvNw6Hw/z8888ufdq2bWvGjx9vjDFm8eLFRpLZv3+/tfzVV1814eHh1vPIyEgzYcKEAmu9mloKIskkJCQUuKxKlSpm+fLlLm3/+7//a1q0aGGMMdY32v/973+3lu/bt89IMt9++60xxphevXqZLl26uGyjb9++xul0Ws8nT57s8npePLaJEydaz0+cOGEcDof55JNPCq0HsCtmiAAU6uGHH9bRo0e1evVqdezYUVu2bNEdd9xhHZJJSkrSiRMnFBoaas0olS1bVikpKfrvf/8rSfr222/VokULl+1e+vxKdu3aJWOMbr/9dpf9bN261dqPJPn7+6tGjRrW84oVKyo9PV3S+RPEjx49qrZt2xa4j6up5Vr88ssvOnz4sAYNGuSyveeffz7f9ho0aOAy5gvjlaTvv/9ed955p0v/S59fzsXbDggIUGBgoLVtAP+Hk6oBXJavr6/at2+v9u3ba9KkSXryySc1efJkDRw4UHl5eapYsaLLuTwXXDjH5WqUKVNGxhiXtrNnz1r/zsvLk4eHh5KSkuTh4eHSr2zZsta/vby8XJY5HA5ru35+fpcdQ3HVcvH2pPOHzZo3b+6y7NIaLh63w+FwWd8YY7VdcOlrdTkFvSYXtg3g/xCIAFyTunXrWpeZ33HHHUpLS5Onp6eioqIK7F+nTh198cUXevzxx622L774wqVPhQoVXE5+zs3N1d69e9WmTRtJUuPGjZWbm6v09HTdc889RRp3YGCgoqKitHHjRmu7F7uaWq5FeHi4KlWqpJ9++kl9+/Yt8nZq166tHTt2uLR9+eWXLs+9vb2Vm5tb5H0AIBABKMTx48f16KOP6oknnlCDBg0UGBioL7/8UrNnz9aDDz4oSWrXrp1atGih7t27a9asWapVq5aOHj2qtWvXqnv37mratKlGjRqlAQMGqGnTpmrVqpXeeecd7du3T9WrV7f2dd999+mZZ57RmjVrVKNGDc2dO1e///67tfz2229X37599fjjj+vFF19U48aN9euvv2rTpk2qX7++OnfufFU1TZkyRUOGDFFYWJjuv/9+ZWdn69///rdGjBhxVbUUJiUlRcnJyS5tt912m6ZMmaKRI0cqKChI999/v3JycvTll18qIyNDzzzzzFWNecSIEbr33ns1Z84cPfDAA9q0aZM++eQTl1mjqKgoawyVK1dWYGCgfHx8rmr7AP5/JXoGEwC3dfr0aTNu3Dhzxx13GKfTafz9/U2tWrXMxIkTzcmTJ61+WVlZZsSIESYyMtJ4eXmZKlWqmL59+5pDhw5ZfaZPn27Kly9vypYtawYMGGCee+45l5OAz5w5Y/785z+bkJAQExYWZmbMmOFyUvWFPpMmTTJRUVHGy8vLREREmIceesjs3r3bGHP+pOqLTzQ2xpiEhARz6a+5hQsXmlq1ahkvLy9TsWJFM2LEiGuq5VKSCnxs3rzZGGPMO++8Yxo1amS8vb1NcHCwuffee83KlSuNMf93UvVXX31lbS8jI8NlfWOMWbRokalUqZLx8/Mz3bt3N88//7yJiIhwea8efvhhU65cOSPJLF682BrbpSd8O51OazmA/+Mw5hoORgNAMZgyZYpWrVqVb1YFV+dPf/qTvvvuO33++eclPRTglsEhMwBwcy+88ILat2+vgIAAffLJJ1q6dGmR7uUEoHAEIgBwczt27NDs2bOVnZ2t6tWr66WXXtKTTz5Z0sMCbikcMgMAALbHjRkBAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDt/X+d7XBEzrL3AwAAAABJRU5ErkJggg=="},"metadata":{}},{"name":"stdout","text":"Max Sequence Length: 1851\nMin Sequence Length: 221\nThe length of selected data: 8835\n","output_type":"stream"}]},{"cell_type":"code","source":"model_id = \"meta-llama/Llama-2-7b-hf\"\naccess_token = \"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\"\n\ntokenizer = AutoTokenizer.from_pretrained(model_id, use_auth_token=access_token)\ntokenizer.pad_token = tokenizer.eos_token\n\nqlora_config = LoraConfig(\n    r=16,\n    lora_alpha=32,\n    lora_dropout=0.05,\n    bias=\"none\",\n    target_modules=[\"q_proj\", \"k_proj\", \"v_proj\", \n                    \"gate_proj\", \"up_proj\", \"down_proj\"],\n    task_type=\"CAUSAL_LM\"\n)\n\nbnb_config = BitsAndBytesConfig(\n    load_in_4bit=True,\n    bnb_4bit_use_double_quant=True,\n    bnb_4bit_quant_type=\"nf4\",\n    bnb_4bit_compute_dtype=torch.bfloat16\n)\n\nmodel = AutoModelForCausalLM.from_pretrained(\n    model_id,\n    quantization_config=bnb_config,\n    trust_remote_code=True,\n    use_auth_token=access_token\n)\n\nmodel.config.use_cache=False\n# model.config.pretraining_tp=1\n\nprint(model)","metadata":{"execution":{"iopub.status.busy":"2023-08-07T12:20:02.586139Z","iopub.execute_input":"2023-08-07T12:20:02.586597Z","iopub.status.idle":"2023-08-07T12:21:17.968044Z","shell.execute_reply.started":"2023-08-07T12:20:02.586558Z","shell.execute_reply":"2023-08-07T12:21:17.967054Z"},"trusted":true},"execution_count":10,"outputs":[{"output_type":"display_data","data":{"text/plain":"Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"73dd81111b9349d0b5f01aa98977aae2"}},"metadata":{}},{"name":"stdout","text":"LlamaForCausalLM(\n  (model): LlamaModel(\n    (embed_tokens): Embedding(32000, 4096, padding_idx=0)\n    (layers): ModuleList(\n      (0-31): 32 x LlamaDecoderLayer(\n        (self_attn): LlamaAttention(\n          (q_proj): Linear4bit(in_features=4096, out_features=4096, bias=False)\n          (k_proj): Linear4bit(in_features=4096, out_features=4096, bias=False)\n          (v_proj): Linear4bit(in_features=4096, out_features=4096, bias=False)\n          (o_proj): Linear4bit(in_features=4096, out_features=4096, bias=False)\n          (rotary_emb): LlamaRotaryEmbedding()\n        )\n        (mlp): LlamaMLP(\n          (gate_proj): Linear4bit(in_features=4096, out_features=11008, bias=False)\n          (down_proj): Linear4bit(in_features=11008, out_features=4096, bias=False)\n          (up_proj): Linear4bit(in_features=4096, out_features=11008, bias=False)\n          (act_fn): SiLUActivation()\n        )\n        (input_layernorm): LlamaRMSNorm()\n        (post_attention_layernorm): LlamaRMSNorm()\n      )\n    )\n    (norm): LlamaRMSNorm()\n  )\n  (lm_head): Linear(in_features=4096, out_features=32000, bias=False)\n)\n","output_type":"stream"}]},{"cell_type":"code","source":"total_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\nprint(\"Total trainable parameters: \", total_params)","metadata":{"execution":{"iopub.status.busy":"2023-08-07T12:21:17.969542Z","iopub.execute_input":"2023-08-07T12:21:17.970008Z","iopub.status.idle":"2023-08-07T12:21:17.977833Z","shell.execute_reply.started":"2023-08-07T12:21:17.969947Z","shell.execute_reply":"2023-08-07T12:21:17.976649Z"},"trusted":true},"execution_count":11,"outputs":[{"name":"stdout","text":"Total trainable parameters:  262410240\n","output_type":"stream"}]},{"cell_type":"code","source":"training_args = TrainingArguments(\n    output_dir=\"./SFT-Llama-7b\", \n    per_device_train_batch_size=2,\n#     per_device_eval_batch_size=2,\n#     gradient_accumulation_steps=2,\n    learning_rate=5e-6,\n    logging_steps=20,\n    logging_strategy=\"steps\",\n#     max_steps=2,\n    num_train_epochs=2,\n    optim=\"paged_adamw_8bit\",\n    fp16=True,\n    run_name=\"baseline-llama-sft\",\n    report_to=\"none\"\n)\n\ntrainer = SFTTrainer(\n    model,\n    train_dataset=data,\n    args=training_args,\n    tokenizer=tokenizer,\n    peft_config=qlora_config,\n    dataset_text_field=\"text\",\n    max_seq_length=2048,\n)","metadata":{"execution":{"iopub.status.busy":"2023-08-07T12:21:17.979638Z","iopub.execute_input":"2023-08-07T12:21:17.980006Z","iopub.status.idle":"2023-08-07T12:22:18.562175Z","shell.execute_reply.started":"2023-08-07T12:21:17.979945Z","shell.execute_reply":"2023-08-07T12:22:18.561212Z"},"trusted":true},"execution_count":12,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/peft/utils/other.py:102: FutureWarning: prepare_model_for_int8_training is deprecated and will be removed in a future version. Use prepare_model_for_kbit_training instead.\n  warnings.warn(\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/9 [00:00<?, ?ba/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e522b4fe6c294cd89b480dfda6ea3ebe"}},"metadata":{}}]},{"cell_type":"code","source":"trainer.train()","metadata":{"execution":{"iopub.status.busy":"2023-08-07T12:22:18.563629Z","iopub.execute_input":"2023-08-07T12:22:18.564003Z","iopub.status.idle":"2023-08-07T12:22:57.489411Z","shell.execute_reply.started":"2023-08-07T12:22:18.563958Z","shell.execute_reply":"2023-08-07T12:22:57.488413Z"},"trusted":true},"execution_count":13,"outputs":[{"name":"stderr","text":"You're using a LlamaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"\n    <div>\n      \n      <progress value='2' max='2' style='width:300px; height:20px; vertical-align: middle;'></progress>\n      [2/2 00:18, Epoch 0/1]\n    </div>\n    <table border=\"1\" class=\"dataframe\">\n  <thead>\n <tr style=\"text-align: left;\">\n      <th>Step</th>\n      <th>Training Loss</th>\n    </tr>\n  </thead>\n  <tbody>\n  </tbody>\n</table><p>"},"metadata":{}},{"execution_count":13,"output_type":"execute_result","data":{"text/plain":"TrainOutput(global_step=2, training_loss=1.8159891366958618, metrics={'train_runtime': 38.5184, 'train_samples_per_second': 0.208, 'train_steps_per_second': 0.052, 'total_flos': 29910605561856.0, 'train_loss': 1.8159891366958618, 'epoch': 0.0})"},"metadata":{}}]},{"cell_type":"code","source":"print(\"Saving The Final Model...\")\ntrainer.save_model(\"./finetuned_llama_7b\")\n# os.makedirs(\"./model\", exist_ok=True)\n# trainer.model.save_pretrained(\"./model\")","metadata":{"execution":{"iopub.status.busy":"2023-08-07T12:22:57.490790Z","iopub.execute_input":"2023-08-07T12:22:57.491262Z","iopub.status.idle":"2023-08-07T12:22:58.085058Z","shell.execute_reply.started":"2023-08-07T12:22:57.491226Z","shell.execute_reply":"2023-08-07T12:22:58.084000Z"},"trusted":true},"execution_count":14,"outputs":[{"name":"stdout","text":"Saving The Final Model...\n","output_type":"stream"}]},{"cell_type":"code","source":"from peft import PeftModel, PeftConfig\nfinetuned_model = AutoModelForCausalLM.from_pretrained(\n    model_id,\n    quantization_config=bnb_config,\n    trust_remote_code=True,\n    use_auth_token=access_token\n)\nfinetuned_model = PeftModel.from_pretrained(finetuned_model, \"./finetuned_llama_7b\")\n# finetuned_model = finetuned_model.merge_and_unload()\nprint(finetuned_model)","metadata":{"execution":{"iopub.status.busy":"2023-08-07T12:22:58.089804Z","iopub.execute_input":"2023-08-07T12:22:58.090133Z","iopub.status.idle":"2023-08-07T12:25:03.031218Z","shell.execute_reply.started":"2023-08-07T12:22:58.090106Z","shell.execute_reply":"2023-08-07T12:25:03.030119Z"},"trusted":true},"execution_count":15,"outputs":[{"output_type":"display_data","data":{"text/plain":"Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"32f84a1acb484bf5b4fb79a3c7a45071"}},"metadata":{}},{"name":"stdout","text":"PeftModelForCausalLM(\n  (base_model): LoraModel(\n    (model): LlamaForCausalLM(\n      (model): LlamaModel(\n        (embed_tokens): Embedding(32000, 4096, padding_idx=0)\n        (layers): ModuleList(\n          (0-31): 32 x LlamaDecoderLayer(\n            (self_attn): LlamaAttention(\n              (q_proj): Linear4bit(\n                in_features=4096, out_features=4096, bias=False\n                (lora_dropout): ModuleDict(\n                  (default): Dropout(p=0.05, inplace=False)\n                )\n                (lora_A): ModuleDict(\n                  (default): Linear(in_features=4096, out_features=16, bias=False)\n                )\n                (lora_B): ModuleDict(\n                  (default): Linear(in_features=16, out_features=4096, bias=False)\n                )\n                (lora_embedding_A): ParameterDict()\n                (lora_embedding_B): ParameterDict()\n              )\n              (k_proj): Linear4bit(\n                in_features=4096, out_features=4096, bias=False\n                (lora_dropout): ModuleDict(\n                  (default): Dropout(p=0.05, inplace=False)\n                )\n                (lora_A): ModuleDict(\n                  (default): Linear(in_features=4096, out_features=16, bias=False)\n                )\n                (lora_B): ModuleDict(\n                  (default): Linear(in_features=16, out_features=4096, bias=False)\n                )\n                (lora_embedding_A): ParameterDict()\n                (lora_embedding_B): ParameterDict()\n              )\n              (v_proj): Linear4bit(\n                in_features=4096, out_features=4096, bias=False\n                (lora_dropout): ModuleDict(\n                  (default): Dropout(p=0.05, inplace=False)\n                )\n                (lora_A): ModuleDict(\n                  (default): Linear(in_features=4096, out_features=16, bias=False)\n                )\n                (lora_B): ModuleDict(\n                  (default): Linear(in_features=16, out_features=4096, bias=False)\n                )\n                (lora_embedding_A): ParameterDict()\n                (lora_embedding_B): ParameterDict()\n              )\n              (o_proj): Linear4bit(in_features=4096, out_features=4096, bias=False)\n              (rotary_emb): LlamaRotaryEmbedding()\n            )\n            (mlp): LlamaMLP(\n              (gate_proj): Linear4bit(\n                in_features=4096, out_features=11008, bias=False\n                (lora_dropout): ModuleDict(\n                  (default): Dropout(p=0.05, inplace=False)\n                )\n                (lora_A): ModuleDict(\n                  (default): Linear(in_features=4096, out_features=16, bias=False)\n                )\n                (lora_B): ModuleDict(\n                  (default): Linear(in_features=16, out_features=11008, bias=False)\n                )\n                (lora_embedding_A): ParameterDict()\n                (lora_embedding_B): ParameterDict()\n              )\n              (down_proj): Linear4bit(\n                in_features=11008, out_features=4096, bias=False\n                (lora_dropout): ModuleDict(\n                  (default): Dropout(p=0.05, inplace=False)\n                )\n                (lora_A): ModuleDict(\n                  (default): Linear(in_features=11008, out_features=16, bias=False)\n                )\n                (lora_B): ModuleDict(\n                  (default): Linear(in_features=16, out_features=4096, bias=False)\n                )\n                (lora_embedding_A): ParameterDict()\n                (lora_embedding_B): ParameterDict()\n              )\n              (up_proj): Linear4bit(\n                in_features=4096, out_features=11008, bias=False\n                (lora_dropout): ModuleDict(\n                  (default): Dropout(p=0.05, inplace=False)\n                )\n                (lora_A): ModuleDict(\n                  (default): Linear(in_features=4096, out_features=16, bias=False)\n                )\n                (lora_B): ModuleDict(\n                  (default): Linear(in_features=16, out_features=11008, bias=False)\n                )\n                (lora_embedding_A): ParameterDict()\n                (lora_embedding_B): ParameterDict()\n              )\n              (act_fn): SiLUActivation()\n            )\n            (input_layernorm): LlamaRMSNorm()\n            (post_attention_layernorm): LlamaRMSNorm()\n          )\n        )\n        (norm): LlamaRMSNorm()\n      )\n      (lm_head): Linear(in_features=4096, out_features=32000, bias=False)\n    )\n  )\n)\n","output_type":"stream"}]},{"cell_type":"code","source":"print(model)","metadata":{"execution":{"iopub.status.busy":"2023-08-07T12:25:03.032820Z","iopub.execute_input":"2023-08-07T12:25:03.033180Z","iopub.status.idle":"2023-08-07T12:25:03.059051Z","shell.execute_reply.started":"2023-08-07T12:25:03.033151Z","shell.execute_reply":"2023-08-07T12:25:03.058032Z"},"trusted":true},"execution_count":16,"outputs":[{"name":"stdout","text":"LlamaForCausalLM(\n  (model): LlamaModel(\n    (embed_tokens): Embedding(32000, 4096, padding_idx=0)\n    (layers): ModuleList(\n      (0-31): 32 x LlamaDecoderLayer(\n        (self_attn): LlamaAttention(\n          (q_proj): Linear4bit(\n            in_features=4096, out_features=4096, bias=False\n            (lora_dropout): ModuleDict(\n              (default): Dropout(p=0.05, inplace=False)\n            )\n            (lora_A): ModuleDict(\n              (default): Linear(in_features=4096, out_features=16, bias=False)\n            )\n            (lora_B): ModuleDict(\n              (default): Linear(in_features=16, out_features=4096, bias=False)\n            )\n            (lora_embedding_A): ParameterDict()\n            (lora_embedding_B): ParameterDict()\n          )\n          (k_proj): Linear4bit(\n            in_features=4096, out_features=4096, bias=False\n            (lora_dropout): ModuleDict(\n              (default): Dropout(p=0.05, inplace=False)\n            )\n            (lora_A): ModuleDict(\n              (default): Linear(in_features=4096, out_features=16, bias=False)\n            )\n            (lora_B): ModuleDict(\n              (default): Linear(in_features=16, out_features=4096, bias=False)\n            )\n            (lora_embedding_A): ParameterDict()\n            (lora_embedding_B): ParameterDict()\n          )\n          (v_proj): Linear4bit(\n            in_features=4096, out_features=4096, bias=False\n            (lora_dropout): ModuleDict(\n              (default): Dropout(p=0.05, inplace=False)\n            )\n            (lora_A): ModuleDict(\n              (default): Linear(in_features=4096, out_features=16, bias=False)\n            )\n            (lora_B): ModuleDict(\n              (default): Linear(in_features=16, out_features=4096, bias=False)\n            )\n            (lora_embedding_A): ParameterDict()\n            (lora_embedding_B): ParameterDict()\n          )\n          (o_proj): Linear4bit(in_features=4096, out_features=4096, bias=False)\n          (rotary_emb): LlamaRotaryEmbedding()\n        )\n        (mlp): LlamaMLP(\n          (gate_proj): Linear4bit(\n            in_features=4096, out_features=11008, bias=False\n            (lora_dropout): ModuleDict(\n              (default): Dropout(p=0.05, inplace=False)\n            )\n            (lora_A): ModuleDict(\n              (default): Linear(in_features=4096, out_features=16, bias=False)\n            )\n            (lora_B): ModuleDict(\n              (default): Linear(in_features=16, out_features=11008, bias=False)\n            )\n            (lora_embedding_A): ParameterDict()\n            (lora_embedding_B): ParameterDict()\n          )\n          (down_proj): Linear4bit(\n            in_features=11008, out_features=4096, bias=False\n            (lora_dropout): ModuleDict(\n              (default): Dropout(p=0.05, inplace=False)\n            )\n            (lora_A): ModuleDict(\n              (default): Linear(in_features=11008, out_features=16, bias=False)\n            )\n            (lora_B): ModuleDict(\n              (default): Linear(in_features=16, out_features=4096, bias=False)\n            )\n            (lora_embedding_A): ParameterDict()\n            (lora_embedding_B): ParameterDict()\n          )\n          (up_proj): Linear4bit(\n            in_features=4096, out_features=11008, bias=False\n            (lora_dropout): ModuleDict(\n              (default): Dropout(p=0.05, inplace=False)\n            )\n            (lora_A): ModuleDict(\n              (default): Linear(in_features=4096, out_features=16, bias=False)\n            )\n            (lora_B): ModuleDict(\n              (default): Linear(in_features=16, out_features=11008, bias=False)\n            )\n            (lora_embedding_A): ParameterDict()\n            (lora_embedding_B): ParameterDict()\n          )\n          (act_fn): SiLUActivation()\n        )\n        (input_layernorm): LlamaRMSNorm()\n        (post_attention_layernorm): LlamaRMSNorm()\n      )\n    )\n    (norm): LlamaRMSNorm()\n  )\n  (lm_head): Linear(in_features=4096, out_features=32000, bias=False)\n)\n","output_type":"stream"}]},{"cell_type":"code","source":"params1 = model.state_dict()\nparams2 = finetuned_model.base_model.model.state_dict()\n\ndef are_models_equal(params1, params2):\n    for key in params2.keys():\n        if key in params1.keys():\n            if not torch.allclose(params1[key].half(), params2[key].half()):\n                return False\n#             else:\n#                 print(params1[key].half())\n#                 print(params2[key].half())\n#                 print(\"Same!!!\")\n        else:\n            print(\"Additional Keys:\", key)\n    return True\n\nif are_models_equal(params1, params2):\n    print(\"They are the same model\")\nelse:\n    print(\"They are not the same model\")","metadata":{"execution":{"iopub.status.busy":"2023-08-07T12:28:42.128761Z","iopub.execute_input":"2023-08-07T12:28:42.129176Z","iopub.status.idle":"2023-08-07T12:28:43.136565Z","shell.execute_reply.started":"2023-08-07T12:28:42.129140Z","shell.execute_reply":"2023-08-07T12:28:43.135338Z"},"trusted":true},"execution_count":27,"outputs":[{"name":"stdout","text":"They are the same model\n","output_type":"stream"}]},{"cell_type":"code","source":"if os.getenv(\"KAGGLE_IS_COMPETITION_RERUN\"):\n    test = pd.read_csv(\"/kaggle/input/kaggle-llm-science-exam/test.csv\", index_col=\"id\")\n    test[\"answer\"] = \"A\"\nelse:\n    test = pd.read_csv(\"/kaggle/input/kaggle-llm-science-exam/train.csv\", index_col=\"id\")\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2023-08-07T12:25:04.097084Z","iopub.execute_input":"2023-08-07T12:25:04.097548Z","iopub.status.idle":"2023-08-07T12:25:04.125246Z","shell.execute_reply.started":"2023-08-07T12:25:04.097511Z","shell.execute_reply":"2023-08-07T12:25:04.124068Z"},"trusted":true},"execution_count":18,"outputs":[{"execution_count":18,"output_type":"execute_result","data":{"text/plain":"                                               prompt  \\\nid                                                      \n0   Which of the following statements accurately d...   \n1   Which of the following is an accurate definiti...   \n2   Which of the following statements accurately d...   \n3   What is the significance of regularization in ...   \n4   Which of the following statements accurately d...   \n\n                                                    A  \\\nid                                                      \n0   MOND is a theory that reduces the observed mis...   \n1   Dynamic scaling refers to the evolution of sel...   \n2   The triskeles symbol was reconstructed as a fe...   \n3   Regularizing the mass-energy of an electron wi...   \n4   The angular spacing of features in the diffrac...   \n\n                                                    B  \\\nid                                                      \n0   MOND is a theory that increases the discrepanc...   \n1   Dynamic scaling refers to the non-evolution of...   \n2   The triskeles symbol is a representation of th...   \n3   Regularizing the mass-energy of an electron wi...   \n4   The angular spacing of features in the diffrac...   \n\n                                                    C  \\\nid                                                      \n0   MOND is a theory that explains the missing bar...   \n1   Dynamic scaling refers to the evolution of sel...   \n2   The triskeles symbol is a representation of a ...   \n3   Regularizing the mass-energy of an electron wi...   \n4   The angular spacing of features in the diffrac...   \n\n                                                    D  \\\nid                                                      \n0   MOND is a theory that reduces the discrepancy ...   \n1   Dynamic scaling refers to the non-evolution of...   \n2   The triskeles symbol represents three interloc...   \n3   Regularizing the mass-energy of an electron wi...   \n4   The angular spacing of features in the diffrac...   \n\n                                                    E answer  \nid                                                            \n0   MOND is a theory that eliminates the observed ...      D  \n1   Dynamic scaling refers to the evolution of sel...      A  \n2   The triskeles symbol is a representation of th...      A  \n3   Regularizing the mass-energy of an electron wi...      C  \n4   The angular spacing of features in the diffrac...      D  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>prompt</th>\n      <th>A</th>\n      <th>B</th>\n      <th>C</th>\n      <th>D</th>\n      <th>E</th>\n      <th>answer</th>\n    </tr>\n    <tr>\n      <th>id</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>Which of the following statements accurately d...</td>\n      <td>MOND is a theory that reduces the observed mis...</td>\n      <td>MOND is a theory that increases the discrepanc...</td>\n      <td>MOND is a theory that explains the missing bar...</td>\n      <td>MOND is a theory that reduces the discrepancy ...</td>\n      <td>MOND is a theory that eliminates the observed ...</td>\n      <td>D</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>Which of the following is an accurate definiti...</td>\n      <td>Dynamic scaling refers to the evolution of sel...</td>\n      <td>Dynamic scaling refers to the non-evolution of...</td>\n      <td>Dynamic scaling refers to the evolution of sel...</td>\n      <td>Dynamic scaling refers to the non-evolution of...</td>\n      <td>Dynamic scaling refers to the evolution of sel...</td>\n      <td>A</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>Which of the following statements accurately d...</td>\n      <td>The triskeles symbol was reconstructed as a fe...</td>\n      <td>The triskeles symbol is a representation of th...</td>\n      <td>The triskeles symbol is a representation of a ...</td>\n      <td>The triskeles symbol represents three interloc...</td>\n      <td>The triskeles symbol is a representation of th...</td>\n      <td>A</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>What is the significance of regularization in ...</td>\n      <td>Regularizing the mass-energy of an electron wi...</td>\n      <td>Regularizing the mass-energy of an electron wi...</td>\n      <td>Regularizing the mass-energy of an electron wi...</td>\n      <td>Regularizing the mass-energy of an electron wi...</td>\n      <td>Regularizing the mass-energy of an electron wi...</td>\n      <td>C</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>Which of the following statements accurately d...</td>\n      <td>The angular spacing of features in the diffrac...</td>\n      <td>The angular spacing of features in the diffrac...</td>\n      <td>The angular spacing of features in the diffrac...</td>\n      <td>The angular spacing of features in the diffrac...</td>\n      <td>The angular spacing of features in the diffrac...</td>\n      <td>D</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"class Perplexity(nn.Module):\n    def __init__(self, reduce: bool = True):\n        super().__init__()\n        self.loss_fn = nn.CrossEntropyLoss()\n        self.reduce = reduce\n\n    def forward(self, logits, labels):\n        shift_logits = logits[..., :-1, :].contiguous()\n        shift_labels = labels[..., 1:].contiguous()\n\n        perplexity = []\n        for i in range(labels.shape[0]):\n            perplexity.append(self.loss_fn(shift_logits[i], shift_labels[i]))\n        perplexity = torch.stack(perplexity, dim=0)\n        if self.reduce:\n            perplexity = torch.mean(perplexity)\n        return perplexity \n    \nperp = Perplexity()","metadata":{"execution":{"iopub.status.busy":"2023-08-07T12:25:04.127476Z","iopub.execute_input":"2023-08-07T12:25:04.127931Z","iopub.status.idle":"2023-08-07T12:25:04.136362Z","shell.execute_reply.started":"2023-08-07T12:25:04.127891Z","shell.execute_reply":"2023-08-07T12:25:04.135225Z"},"trusted":true},"execution_count":19,"outputs":[]},{"cell_type":"code","source":"def precision_at_k(r, k):\n    assert k <= len(r)\n    assert k != 0\n    return sum(int(x) for x in r[:k]) / k\n\ndef MAP_at_3(predictions, true_items):\n    U = len(predictions)\n    map_at_3 = 0.0\n    for u in range(U):\n        user_preds = predictions[u]\n        user_true = true_items[u]\n        user_results = [1 if item == user_true else 0 for item in user_preds]\n        for k in range(min(len(user_preds), 3)):\n            map_at_3 += precision_at_k(user_results, k + 1) * user_results[k]\n    return map_at_3 / U\n\nmaps = []\npreds = []\nfor idx, row in tqdm(test.iterrows(), total=len(test)):\n    with torch.no_grad():\n        cols = [\"A\", \"B\", \"C\", \"D\", \"E\"]\n        perps = []\n        samples = []\n        for col in cols:\n            samples.append(\"<|question|>\" + row[\"prompt\"] + \\\n                           \"</s><|answer|>\" + \"answer: \" + row[col])\n        inputs = tokenizer(samples, return_tensors=\"pt\", \n                           add_special_tokens=False, padding=True, \n                           truncation=True).to(\"cuda\")\n        output = model(input_ids=inputs[\"input_ids\"], \n                       attention_mask=inputs[\"attention_mask\"])\n        output = output.logits\n        labels = inputs[\"input_ids\"]\n        labels.masked_fill_(~inputs[\"attention_mask\"].bool(), -100)\n        for j in range(len(cols)):\n            p = perp(output[j].unsqueeze(0), labels[j].unsqueeze(0))\n            perps.append(p.detach().cpu())\n            \n        del inputs\n        del labels\n        del output\n        del p\n\n    perps = np.array(perps)\n        \n    predictions = [np.array(cols)[np.argsort(perps)]]\n    preds.append(predictions)\n    tp = [row.answer]\n    map = MAP_at_3(predictions, tp)\n    maps.append(map)\n    print(np.mean(maps))","metadata":{"execution":{"iopub.status.busy":"2023-08-07T12:25:04.138194Z","iopub.execute_input":"2023-08-07T12:25:04.138773Z","iopub.status.idle":"2023-08-07T12:26:40.611902Z","shell.execute_reply.started":"2023-08-07T12:25:04.138732Z","shell.execute_reply":"2023-08-07T12:26:40.609931Z"},"trusted":true},"execution_count":20,"outputs":[{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/200 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"35820ea657af40d182e047152a4be2be"}},"metadata":{}},{"name":"stderr","text":"Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.\n/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:31: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n  warnings.warn(\"None of the inputs have requires_grad=True. Gradients will be None\")\n","output_type":"stream"},{"name":"stdout","text":"0.5\n0.41666666666666663\n0.27777777777777773\n0.4583333333333333\n0.4666666666666666\n0.38888888888888884\n0.3333333333333333\n0.29166666666666663\n0.2962962962962963\n0.31666666666666665\n0.28787878787878785\n0.3055555555555555\n0.28205128205128205\n0.2857142857142857\n0.26666666666666666\n0.25\n0.23529411764705882\n0.25\n0.2894736842105263\n0.325\n0.3333333333333333\n0.36363636363636365\n0.391304347826087\n0.375\n0.38\n0.40384615384615385\n0.42592592592592593\n0.42857142857142855\n0.43103448275862066\n0.45\n0.46774193548387094\n0.453125\n","output_type":"stream"},{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)","Cell \u001b[0;32mIn[20], line 30\u001b[0m\n\u001b[1;32m     25\u001b[0m     samples\u001b[38;5;241m.\u001b[39mappend(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m<|question|>\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m row[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mprompt\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m+\u001b[39m \\\n\u001b[1;32m     26\u001b[0m                    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m</s><|answer|>\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124manswer: \u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m row[col])\n\u001b[1;32m     27\u001b[0m inputs \u001b[38;5;241m=\u001b[39m tokenizer(samples, return_tensors\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpt\u001b[39m\u001b[38;5;124m\"\u001b[39m, \n\u001b[1;32m     28\u001b[0m                    add_special_tokens\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m, padding\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, \n\u001b[1;32m     29\u001b[0m                    truncation\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\u001b[38;5;241m.\u001b[39mto(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcuda\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m---> 30\u001b[0m output \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\u001b[43minput_ids\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43minput_ids\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m     31\u001b[0m \u001b[43m               \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mattention_mask\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     32\u001b[0m output \u001b[38;5;241m=\u001b[39m output\u001b[38;5;241m.\u001b[39mlogits\n\u001b[1;32m     33\u001b[0m labels \u001b[38;5;241m=\u001b[39m inputs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minput_ids\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1499\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1500\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/accelerate/hooks.py:165\u001b[0m, in \u001b[0;36madd_hook_to_module.<locals>.new_forward\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    163\u001b[0m         output \u001b[38;5;241m=\u001b[39m old_forward(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m    164\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 165\u001b[0m     output \u001b[38;5;241m=\u001b[39m \u001b[43mold_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    166\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m module\u001b[38;5;241m.\u001b[39m_hf_hook\u001b[38;5;241m.\u001b[39mpost_forward(module, output)\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py:688\u001b[0m, in \u001b[0;36mLlamaForCausalLM.forward\u001b[0;34m(self, input_ids, attention_mask, position_ids, past_key_values, inputs_embeds, labels, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m    685\u001b[0m return_dict \u001b[38;5;241m=\u001b[39m return_dict \u001b[38;5;28;01mif\u001b[39;00m return_dict \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39muse_return_dict\n\u001b[1;32m    687\u001b[0m \u001b[38;5;66;03m# decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)\u001b[39;00m\n\u001b[0;32m--> 688\u001b[0m outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    689\u001b[0m \u001b[43m    \u001b[49m\u001b[43minput_ids\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minput_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    690\u001b[0m \u001b[43m    \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    691\u001b[0m \u001b[43m    \u001b[49m\u001b[43mposition_ids\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mposition_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    692\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpast_key_values\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpast_key_values\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    693\u001b[0m \u001b[43m    \u001b[49m\u001b[43minputs_embeds\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs_embeds\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    694\u001b[0m \u001b[43m    \u001b[49m\u001b[43muse_cache\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cache\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    695\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    696\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    697\u001b[0m \u001b[43m    \u001b[49m\u001b[43mreturn_dict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreturn_dict\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    698\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    700\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m    701\u001b[0m logits \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlm_head(hidden_states)\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1499\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1500\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/accelerate/hooks.py:165\u001b[0m, in \u001b[0;36madd_hook_to_module.<locals>.new_forward\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    163\u001b[0m         output \u001b[38;5;241m=\u001b[39m old_forward(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m    164\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 165\u001b[0m     output \u001b[38;5;241m=\u001b[39m \u001b[43mold_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    166\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m module\u001b[38;5;241m.\u001b[39m_hf_hook\u001b[38;5;241m.\u001b[39mpost_forward(module, output)\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py:570\u001b[0m, in \u001b[0;36mLlamaModel.forward\u001b[0;34m(self, input_ids, attention_mask, position_ids, past_key_values, inputs_embeds, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m    566\u001b[0m             \u001b[38;5;28;01mreturn\u001b[39;00m module(\u001b[38;5;241m*\u001b[39minputs, output_attentions, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[1;32m    568\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m custom_forward\n\u001b[0;32m--> 570\u001b[0m     layer_outputs \u001b[38;5;241m=\u001b[39m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mutils\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcheckpoint\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcheckpoint\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    571\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcreate_custom_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdecoder_layer\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    572\u001b[0m \u001b[43m        \u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    573\u001b[0m \u001b[43m        \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    574\u001b[0m \u001b[43m        \u001b[49m\u001b[43mposition_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    575\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    576\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    577\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    578\u001b[0m     layer_outputs \u001b[38;5;241m=\u001b[39m decoder_layer(\n\u001b[1;32m    579\u001b[0m         hidden_states,\n\u001b[1;32m    580\u001b[0m         attention_mask\u001b[38;5;241m=\u001b[39mattention_mask,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    584\u001b[0m         use_cache\u001b[38;5;241m=\u001b[39muse_cache,\n\u001b[1;32m    585\u001b[0m     )\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:249\u001b[0m, in \u001b[0;36mcheckpoint\u001b[0;34m(function, use_reentrant, *args, **kwargs)\u001b[0m\n\u001b[1;32m    246\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUnexpected keyword arguments: \u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(arg \u001b[38;5;28;01mfor\u001b[39;00m arg \u001b[38;5;129;01min\u001b[39;00m kwargs))\n\u001b[1;32m    248\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m use_reentrant:\n\u001b[0;32m--> 249\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mCheckpointFunction\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mapply\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfunction\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpreserve\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    250\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    251\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m _checkpoint_without_reentrant(\n\u001b[1;32m    252\u001b[0m         function,\n\u001b[1;32m    253\u001b[0m         preserve,\n\u001b[1;32m    254\u001b[0m         \u001b[38;5;241m*\u001b[39margs,\n\u001b[1;32m    255\u001b[0m         \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs,\n\u001b[1;32m    256\u001b[0m     )\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/torch/autograd/function.py:506\u001b[0m, in \u001b[0;36mFunction.apply\u001b[0;34m(cls, *args, **kwargs)\u001b[0m\n\u001b[1;32m    503\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m torch\u001b[38;5;241m.\u001b[39m_C\u001b[38;5;241m.\u001b[39m_are_functorch_transforms_active():\n\u001b[1;32m    504\u001b[0m     \u001b[38;5;66;03m# See NOTE: [functorch vjp and autograd interaction]\u001b[39;00m\n\u001b[1;32m    505\u001b[0m     args \u001b[38;5;241m=\u001b[39m _functorch\u001b[38;5;241m.\u001b[39mutils\u001b[38;5;241m.\u001b[39munwrap_dead_wrappers(args)\n\u001b[0;32m--> 506\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mapply\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m    508\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39msetup_context \u001b[38;5;241m==\u001b[39m _SingleLevelFunction\u001b[38;5;241m.\u001b[39msetup_context:\n\u001b[1;32m    509\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[1;32m    510\u001b[0m         \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mIn order to use an autograd.Function with functorch transforms \u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m    511\u001b[0m         \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m(vmap, grad, jvp, jacrev, ...), it must override the setup_context \u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m    512\u001b[0m         \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mstaticmethod. For more details, please see \u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m    513\u001b[0m         \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhttps://pytorch.org/docs/master/notes/extending.func.html\u001b[39m\u001b[38;5;124m'\u001b[39m)\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:107\u001b[0m, in \u001b[0;36mCheckpointFunction.forward\u001b[0;34m(ctx, run_function, preserve_rng_state, *args)\u001b[0m\n\u001b[1;32m    104\u001b[0m ctx\u001b[38;5;241m.\u001b[39msave_for_backward(\u001b[38;5;241m*\u001b[39mtensor_inputs)\n\u001b[1;32m    106\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mno_grad():\n\u001b[0;32m--> 107\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m \u001b[43mrun_function\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    108\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m outputs\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py:566\u001b[0m, in \u001b[0;36mLlamaModel.forward.<locals>.create_custom_forward.<locals>.custom_forward\u001b[0;34m(*inputs)\u001b[0m\n\u001b[1;32m    564\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcustom_forward\u001b[39m(\u001b[38;5;241m*\u001b[39minputs):\n\u001b[1;32m    565\u001b[0m     \u001b[38;5;66;03m# None for past_key_value\u001b[39;00m\n\u001b[0;32m--> 566\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mmodule\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1499\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1500\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/accelerate/hooks.py:165\u001b[0m, in \u001b[0;36madd_hook_to_module.<locals>.new_forward\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    163\u001b[0m         output \u001b[38;5;241m=\u001b[39m old_forward(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m    164\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 165\u001b[0m     output \u001b[38;5;241m=\u001b[39m \u001b[43mold_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    166\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m module\u001b[38;5;241m.\u001b[39m_hf_hook\u001b[38;5;241m.\u001b[39mpost_forward(module, output)\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py:292\u001b[0m, in \u001b[0;36mLlamaDecoderLayer.forward\u001b[0;34m(self, hidden_states, attention_mask, position_ids, past_key_value, output_attentions, use_cache)\u001b[0m\n\u001b[1;32m    289\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minput_layernorm(hidden_states)\n\u001b[1;32m    291\u001b[0m \u001b[38;5;66;03m# Self Attention\u001b[39;00m\n\u001b[0;32m--> 292\u001b[0m hidden_states, self_attn_weights, present_key_value \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mself_attn\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    293\u001b[0m \u001b[43m    \u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mhidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    294\u001b[0m \u001b[43m    \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    295\u001b[0m \u001b[43m    \u001b[49m\u001b[43mposition_ids\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mposition_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    296\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpast_key_value\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpast_key_value\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    297\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    298\u001b[0m \u001b[43m    \u001b[49m\u001b[43muse_cache\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cache\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    299\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    300\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m residual \u001b[38;5;241m+\u001b[39m hidden_states\n\u001b[1;32m    302\u001b[0m \u001b[38;5;66;03m# Fully Connected\u001b[39;00m\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1499\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1500\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/accelerate/hooks.py:165\u001b[0m, in \u001b[0;36madd_hook_to_module.<locals>.new_forward\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    163\u001b[0m         output \u001b[38;5;241m=\u001b[39m old_forward(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m    164\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 165\u001b[0m     output \u001b[38;5;241m=\u001b[39m \u001b[43mold_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    166\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m module\u001b[38;5;241m.\u001b[39m_hf_hook\u001b[38;5;241m.\u001b[39mpost_forward(module, output)\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py:227\u001b[0m, in \u001b[0;36mLlamaAttention.forward\u001b[0;34m(self, hidden_states, attention_mask, position_ids, past_key_value, output_attentions, use_cache)\u001b[0m\n\u001b[1;32m    222\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m    223\u001b[0m             \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAttention mask should be of size \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m(bsz,\u001b[38;5;250m \u001b[39m\u001b[38;5;241m1\u001b[39m,\u001b[38;5;250m \u001b[39mq_len,\u001b[38;5;250m \u001b[39mkv_seq_len)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, but is \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mattention_mask\u001b[38;5;241m.\u001b[39msize()\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    224\u001b[0m         )\n\u001b[1;32m    225\u001b[0m     attn_weights \u001b[38;5;241m=\u001b[39m attn_weights \u001b[38;5;241m+\u001b[39m attention_mask\n\u001b[1;32m    226\u001b[0m     attn_weights \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mmax(\n\u001b[0;32m--> 227\u001b[0m         attn_weights, \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtensor\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfinfo\u001b[49m\u001b[43m(\u001b[49m\u001b[43mattn_weights\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdtype\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmin\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdevice\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattn_weights\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdevice\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    228\u001b[0m     )\n\u001b[1;32m    230\u001b[0m \u001b[38;5;66;03m# upcast attention to fp32\u001b[39;00m\n\u001b[1;32m    231\u001b[0m attn_weights \u001b[38;5;241m=\u001b[39m nn\u001b[38;5;241m.\u001b[39mfunctional\u001b[38;5;241m.\u001b[39msoftmax(attn_weights, dim\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m, dtype\u001b[38;5;241m=\u001b[39mtorch\u001b[38;5;241m.\u001b[39mfloat32)\u001b[38;5;241m.\u001b[39mto(query_states\u001b[38;5;241m.\u001b[39mdtype)\n","\u001b[0;31mKeyboardInterrupt\u001b[0m: "],"ename":"KeyboardInterrupt","evalue":"","output_type":"error"}]},{"cell_type":"code","source":"submission = pd.read_csv(\"/kaggle/input/kaggle-llm-science-exam/sample_submission.csv\")\nsubmission[\"prediction\"] = [\" \".join(p[0][:3]) for p in preds]","metadata":{"execution":{"iopub.status.busy":"2023-08-07T12:26:40.613048Z","iopub.status.idle":"2023-08-07T12:26:40.614188Z","shell.execute_reply.started":"2023-08-07T12:26:40.613891Z","shell.execute_reply":"2023-08-07T12:26:40.613917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.head()","metadata":{"execution":{"iopub.status.busy":"2023-08-07T12:26:40.615530Z","iopub.status.idle":"2023-08-07T12:26:40.616537Z","shell.execute_reply.started":"2023-08-07T12:26:40.616255Z","shell.execute_reply":"2023-08-07T12:26:40.616281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-08-07T12:26:40.617922Z","iopub.status.idle":"2023-08-07T12:26:40.619011Z","shell.execute_reply.started":"2023-08-07T12:26:40.618742Z","shell.execute_reply":"2023-08-07T12:26:40.618769Z"},"trusted":true},"execution_count":null,"outputs":[]}]}