detect-fake-text-keras-nlp-train.ipynb

{"metadata":{"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":61542,"databundleVersionId":6888007,"sourceType":"competition"},{"sourceId":6847931,"sourceType":"datasetVersion","datasetId":3936750},{"sourceId":6867914,"sourceType":"datasetVersion","datasetId":3946973},{"sourceId":6890527,"sourceType":"datasetVersion","datasetId":3942644}],"dockerImageVersionId":30588,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.12"},"papermill":{"default_parameters":{},"duration":17006.067104,"end_time":"2023-11-17T07:34:19.159496","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2023-11-17T02:50:53.092392","version":"2.4.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# LLM - Detect AI Generated Text\n> Identify which essay was written by a large language model\n\n<img src=\"https://user-images.githubusercontent.com/36858976/279902422-b365f6ef-ef01-49ac-af7f-0bc2ca3ba835.png\">","metadata":{"papermill":{"duration":0.017135,"end_time":"2023-11-17T02:50:57.930783","exception":false,"start_time":"2023-11-17T02:50:57.913648","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# 🎯 | Motivation\n\n* In this notebook, we will demonstrate the usage of the multi-backend capabilities of `KerasCore` and `KerasNLP` for the **Detecting Fake Text** task.\n* Thanks to `KerasCore`, soon to become `Keras 3.0`, which enables seamless execution of this notebook on `TensorFlow`, `Jax`, and `PyTorch` platforms with minimal adjustments required.\n* Moreover, this notebook supports both single/multi GPU and TPU training. As time progresses, larger datasets may become available, making TPUs invaluable for training substantial models on these extensive datasets.\n* Finally, I am very curious about the effectiveness of this competition, as a few months ago, `OpenAI` shut down their AI Text-Detection Tool due to inaccuracies ([ref](https://www.pcmag.com/news/openai-quietly-shuts-down-ai-text-detection-tool-over-inaccuracies)). So, I'm eager to see how well this competition will address the problem.","metadata":{"papermill":{"duration":0.016184,"end_time":"2023-11-17T02:50:57.963895","exception":false,"start_time":"2023-11-17T02:50:57.947711","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# 🛠 | Install Libraries ","metadata":{"papermill":{"duration":0.017175,"end_time":"2023-11-17T02:50:58.031078","exception":false,"start_time":"2023-11-17T02:50:58.013903","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install -q keras-core --upgrade\n!pip install -q keras-nlp --upgrade\n!pip install --upgrade -q wandb git+https://github.com/soumik12345/wandb-addons","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":65.396742,"end_time":"2023-11-17T02:52:03.444927","exception":false,"start_time":"2023-11-17T02:50:58.048185","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-28T05:15:33.076640Z","iopub.execute_input":"2023-11-28T05:15:33.077333Z","iopub.status.idle":"2023-11-28T05:16:35.254610Z","shell.execute_reply.started":"2023-11-28T05:15:33.077301Z","shell.execute_reply":"2023-11-28T05:16:35.253412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📚 | Import Libraries ","metadata":{"papermill":{"duration":0.016051,"end_time":"2023-11-17T02:52:03.477738","exception":false,"start_time":"2023-11-17T02:52:03.461687","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nos.environ[\"KERAS_BACKEND\"] = \"torch\"  # \"jax\" or \"tensorflow\" or \"torch\" \n# os.environ[\"WANDB_SILENT\"] = \"false\" # for wandb\n\nimport keras_nlp\nimport keras_core as keras\nimport keras_core.backend as K\n\n\nimport torch\n# import jax\nimport tensorflow as tf\n# from tensorflow import keras\n# import tensorflow.keras.backend as K\n\nimport numpy as np \nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nimport matplotlib as mpl\n\ncmap = mpl.cm.get_cmap('coolwarm')","metadata":{"_kg_hide-output":true,"papermill":{"duration":27.415999,"end_time":"2023-11-17T02:52:30.909923","exception":false,"start_time":"2023-11-17T02:52:03.493924","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-28T05:16:37.606746Z","iopub.execute_input":"2023-11-28T05:16:37.607137Z","iopub.status.idle":"2023-11-28T05:16:55.360340Z","shell.execute_reply.started":"2023-11-28T05:16:37.607102Z","shell.execute_reply":"2023-11-28T05:16:55.359325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Library Version","metadata":{"papermill":{"duration":0.016819,"end_time":"2023-11-17T02:52:30.944500","exception":false,"start_time":"2023-11-17T02:52:30.927681","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(\"TensorFlow:\", tf.__version__)\n# print(\"JAX:\", jax.__version__)\nprint(\"Keras:\", keras.__version__)\nprint(\"KerasNLP:\", keras_nlp.__version__)","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.026686,"end_time":"2023-11-17T02:52:30.988184","exception":false,"start_time":"2023-11-17T02:52:30.961498","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-28T05:16:55.362431Z","iopub.execute_input":"2023-11-28T05:16:55.363203Z","iopub.status.idle":"2023-11-28T05:16:55.368928Z","shell.execute_reply.started":"2023-11-28T05:16:55.363165Z","shell.execute_reply":"2023-11-28T05:16:55.368023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ⚙️ | Configuration","metadata":{"papermill":{"duration":0.018648,"end_time":"2023-11-17T02:52:31.025499","exception":false,"start_time":"2023-11-17T02:52:31.006851","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class CFG:\n    verbose = 0  # Verbosity\n    \n    wandb = True  # Weights & Biases logging\n    competition = 'llm-detect-ai-generated-text'  # Competition name\n    _wandb_kernel = 'awsaf49'  # WandB kernel\n    comment = 'DebertaV3-MaxSeq_200-ext_s-torch'  # Comment description\n    \n    preset = \"deberta_v3_base_en\"  # Name of pretrained models\n    sequence_length = 200  # Input sequence length\n    \n    device = 'TPU'  # Device\n    \n    seed = 42  # Random seed\n    \n    num_folds = 5  # Total folds\n    selected_folds = [0, 1, 2]  # Folds to train on\n    \n    epochs = 3 # Training epochs\n    batch_size = 3  # Batch size\n    drop_remainder = True  # Drop incomplete batches\n    cache = True # Caches data after one iteration, use only with `TPU` to avoid OOM\n    \n    scheduler = 'cosine'  # Learning rate scheduler\n    \n    class_names = [\"real\", \"fake\"]  # Class names [A, B, C, D, E]\n    num_classes = len(class_names)  # Number of classes\n    class_labels = list(range(num_classes))  # Class labels [0, 1, 2, 3, 4]\n    label2name = dict(zip(class_labels, class_names))  # Label to class name mapping\n    name2label = {v: k for k, v in label2name.items()}  # Class name to label mapping","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.084973,"end_time":"2023-11-17T02:52:31.128794","exception":false,"start_time":"2023-11-17T02:52:31.043821","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-28T05:17:05.020518Z","iopub.execute_input":"2023-11-28T05:17:05.020898Z","iopub.status.idle":"2023-11-28T05:17:05.030091Z","shell.execute_reply.started":"2023-11-28T05:17:05.020869Z","shell.execute_reply":"2023-11-28T05:17:05.029048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ♻️ | Reproducibility \nSets value for random seed to produce similar result in each run.","metadata":{"papermill":{"duration":0.017469,"end_time":"2023-11-17T02:52:31.164708","exception":false,"start_time":"2023-11-17T02:52:31.147239","status":"completed"},"tags":[]}},{"cell_type":"code","source":"keras.utils.set_random_seed(CFG.seed)","metadata":{"papermill":{"duration":0.034998,"end_time":"2023-11-17T02:52:31.217250","exception":false,"start_time":"2023-11-17T02:52:31.182252","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-28T05:17:06.495018Z","iopub.execute_input":"2023-11-28T05:17:06.495407Z","iopub.status.idle":"2023-11-28T05:17:06.505966Z","shell.execute_reply.started":"2023-11-28T05:17:06.495377Z","shell.execute_reply":"2023-11-28T05:17:06.505050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 💾 | Hardware\nFollowing codes automatically detects hardware (TPU or GPU). ","metadata":{"papermill":{"duration":0.017456,"end_time":"2023-11-17T02:52:31.252730","exception":false,"start_time":"2023-11-17T02:52:31.235274","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_device():\n    \"Detect and intializes GPU/TPU automatically\"\n    try:\n        # Connect to TPU\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect() \n        # Set TPU strategy\n        strategy = tf.distribute.TPUStrategy(tpu)\n        print(f'> Running on TPU', tpu.master(), end=' | ')\n        print('Num of TPUs: ', strategy.num_replicas_in_sync)\n        device=CFG.device\n    except:\n        # If TPU is not available, detect GPUs\n        gpus = tf.config.list_logical_devices('GPU')\n        ngpu = len(gpus)\n         # Check number of GPUs\n        if ngpu:\n            # Set GPU strategy\n            strategy = tf.distribute.MirroredStrategy(gpus) # single-GPU or multi-GPU\n            # Print GPU details\n            print(\"> Running on GPU\", end=' | ')\n            print(\"Num of GPUs: \", ngpu)\n            device='GPU'\n        else:\n            # If no GPUs are available, use CPU\n            print(\"> Running on CPU\")\n            strategy = tf.distribute.get_strategy()\n            device='CPU'\n    return strategy, device","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.028211,"end_time":"2023-11-17T02:52:31.298403","exception":false,"start_time":"2023-11-17T02:52:31.270192","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-28T05:17:08.795512Z","iopub.execute_input":"2023-11-28T05:17:08.795898Z","iopub.status.idle":"2023-11-28T05:17:08.803651Z","shell.execute_reply.started":"2023-11-28T05:17:08.795866Z","shell.execute_reply":"2023-11-28T05:17:08.802745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize GPU/TPU/TPU-VM\nstrategy, CFG.device = get_device()\nCFG.replicas = strategy.num_replicas_in_sync","metadata":{"papermill":{"duration":10.108771,"end_time":"2023-11-17T02:52:41.424602","exception":false,"start_time":"2023-11-17T02:52:31.315831","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-28T05:17:09.666276Z","iopub.execute_input":"2023-11-28T05:17:09.667260Z","iopub.status.idle":"2023-11-28T05:17:17.352948Z","shell.execute_reply.started":"2023-11-28T05:17:09.667219Z","shell.execute_reply":"2023-11-28T05:17:17.351893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📁 | Dataset Path ","metadata":{"papermill":{"duration":0.019278,"end_time":"2023-11-17T02:52:41.463458","exception":false,"start_time":"2023-11-17T02:52:41.444180","status":"completed"},"tags":[]}},{"cell_type":"code","source":"BASE_PATH = '/kaggle/input/llm-detect-ai-generated-text'","metadata":{"papermill":{"duration":0.027916,"end_time":"2023-11-17T02:52:41.510443","exception":false,"start_time":"2023-11-17T02:52:41.482527","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-28T05:17:17.354506Z","iopub.execute_input":"2023-11-28T05:17:17.354842Z","iopub.status.idle":"2023-11-28T05:17:17.379699Z","shell.execute_reply.started":"2023-11-28T05:17:17.354815Z","shell.execute_reply":"2023-11-28T05:17:17.378762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📖 | Meta Data \n* `{test|train}_essays.csv`\n    * `id` - A unique identifier for each essay.\n    * `prompt_id` - Identifies the prompt the essay was written in response to.\n    * `text` - The essay text itself.\n    * `generated` - Whether the essay was written by a student (0) or generated by an LLM (1). This field is the target and is not present in test_essays.csv.\n* **sample_submission.csv** - is the valid sample submission.","metadata":{"papermill":{"duration":0.017686,"end_time":"2023-11-17T02:52:41.547710","exception":false,"start_time":"2023-11-17T02:52:41.530024","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Train Data","metadata":{"papermill":{"duration":0.018099,"end_time":"2023-11-17T02:52:41.584108","exception":false,"start_time":"2023-11-17T02:52:41.566009","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df = pd.read_csv(f'{BASE_PATH}/train_essays.csv')  # Read CSV file into a DataFrame\ndf['label'] = df.generated.copy()\ndf['name'] = df.generated.map(CFG.label2name)  # Map answer labels using name-to-label mapping\n\n# Display information about the train data\nprint(\"# Train Data: {:,}\".format(len(df)))\nprint(\"# Sample:\")\ndisplay(df.head(2))\n\n# Show distribution of answers using a bar plot\nplt.figure(figsize=(8, 4))\ndf.name.value_counts().plot.bar(color=[cmap(0.0), cmap(0.25), cmap(0.65), cmap(0.9), cmap(1.0)])\nplt.xlabel(\"Class\")\nplt.ylabel(\"Count\")\nplt.title(\"Class distribution for Train Data\")\nplt.show()\n","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.505411,"end_time":"2023-11-17T02:52:42.108407","exception":false,"start_time":"2023-11-17T02:52:41.602996","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-28T05:17:17.381107Z","iopub.execute_input":"2023-11-28T05:17:17.381407Z","iopub.status.idle":"2023-11-28T05:17:17.811604Z","shell.execute_reply.started":"2023-11-28T05:17:17.381380Z","shell.execute_reply":"2023-11-28T05:17:17.810726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## External Datasets\n\nWe'll be utilizing the following external datasets:\n\n* [Proper Train Dataset](https://www.kaggle.com/datasets/thedrcat/daigt-proper-train-dataset/) @thedrcat\n* [ArguGPT](https://www.kaggle.com/datasets/alejopaullier/argugpt) @alejopaullier\n","metadata":{"papermill":{"duration":0.01813,"end_time":"2023-11-17T02:52:42.145676","exception":false,"start_time":"2023-11-17T02:52:42.127546","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Load external data\next_df1 = pd.read_csv('/kaggle/input/daigt-proper-train-dataset/train_drcat_04.csv')\next_df2 = pd.read_csv('/kaggle/input/argugpt/argugpt.csv')[['id','text','model']]\n\next_df2.rename(columns={'model':'source'}, inplace=True)\next_df2['label'] = 1\n\next_df = pd.concat([\n    ext_df1[ext_df1.source=='persuade_corpus'].sample(10000),\n    ext_df1[ext_df1.source!='persuade_corpus'],\n#     ext_df2,\n])\n\n# ext_real_df = ext_df[['id', 'text']].copy()\n# ext_real_df['label']  = 0\n\n# ext_fake_df = ext_df[['id', 'source_text']].copy()\n# ext_fake_df.rename(columns={\"source_text\":\"text\"}, inplace=True)\n# ext_fake_df['label']  = 1\n\n# ext_df = pd.concat([ext_real_df, ext_fake_df], axis=0)\next_df['name'] = ext_df.label.map(CFG.label2name)\n\n# Display information about the external data\nprint(\"# External Data: {:,}\".format(len(ext_df)))\nprint(\"# Sample:\")\next_df.head(2)\n\n# Show distribution of answers using a bar plot\nplt.figure(figsize=(8, 4))\next_df.name.value_counts().plot.bar(color=[cmap(0.0), cmap(0.65)])\nplt.xlabel(\"Class\")\nplt.ylabel(\"Count\")\nplt.title(\"Answer distribution for External Data\")\nplt.show()","metadata":{"_kg_hide-input":true,"papermill":{"duration":3.273348,"end_time":"2023-11-17T02:52:45.436656","exception":false,"start_time":"2023-11-17T02:52:42.163308","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-28T05:17:17.813448Z","iopub.execute_input":"2023-11-28T05:17:17.813749Z","iopub.status.idle":"2023-11-28T05:17:20.509992Z","shell.execute_reply.started":"2023-11-28T05:17:17.813722Z","shell.execute_reply":"2023-11-28T05:17:20.509056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Combine External and Train Data","metadata":{"papermill":{"duration":0.018436,"end_time":"2023-11-17T02:52:45.474238","exception":false,"start_time":"2023-11-17T02:52:45.455802","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df = ext_df.copy().reset_index(drop=True) # pd.concat([ext_df, df], axis=0)\ndf.head()","metadata":{"papermill":{"duration":0.044456,"end_time":"2023-11-17T02:52:45.537222","exception":false,"start_time":"2023-11-17T02:52:45.492766","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-28T05:17:20.511322Z","iopub.execute_input":"2023-11-28T05:17:20.511680Z","iopub.status.idle":"2023-11-28T05:17:20.533300Z","shell.execute_reply.started":"2023-11-28T05:17:20.511647Z","shell.execute_reply":"2023-11-28T05:17:20.532367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🔪 | Data Split\n\nIn the code snippet provided below, we will divide the existing **train** data into folds using a stratification of `label` column.","metadata":{"papermill":{"duration":0.019506,"end_time":"2023-11-17T02:52:45.576835","exception":false,"start_time":"2023-11-17T02:52:45.557329","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold  # Import package\n\nskf = StratifiedKFold(n_splits=CFG.num_folds, shuffle=True, random_state=CFG.seed)  # Initialize K-Fold\n\ndf = df.reset_index(drop=True)  # Reset dataframe index\n\ndf['stratify'] = df.label.astype(str)+df.source.astype(str)\n\ndf[\"fold\"] = -1  # New 'fold' column\n\n# Assign folds using StratifiedKFold\nfor fold, (train_idx, val_idx) in enumerate(skf.split(df, df['stratify'])):\n    df.loc[val_idx, 'fold'] = fold\n\n# Display label distribution for each fold\ndf.groupby([\"fold\", \"name\", \"source\"]).size()","metadata":{"_kg_hide-input":true,"papermill":{"duration":1.702934,"end_time":"2023-11-17T02:52:47.300652","exception":false,"start_time":"2023-11-17T02:52:45.597718","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-28T05:17:21.597350Z","iopub.execute_input":"2023-11-28T05:17:21.597896Z","iopub.status.idle":"2023-11-28T05:17:22.386592Z","shell.execute_reply.started":"2023-11-28T05:17:21.597864Z","shell.execute_reply":"2023-11-28T05:17:22.385551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🍽️ | Preprocessing\n\n**What it does:** The preprocessor takes input strings and transforms them into a dictionary (`token_ids`, `padding_mask`) containing preprocessed tensors. This process starts with tokenization, where input strings are converted into sequences of token IDs.\n\n**Why it's important:** Initially, raw text data is complex and challenging for modeling due to its high dimensionality. By converting text into a compact set of tokens, such as transforming `\"The quick brown fox\"` into `[\"the\", \"qu\", \"##ick\", \"br\", \"##own\", \"fox\"]`, we simplify the data. Many models rely on special tokens and additional tensors to understand input. These tokens help divide input and identify padding, among other tasks. Making all sequences the same length through padding boosts computational efficiency, making subsequent steps smoother.\n\nExplore the following pages to access the available preprocessing and tokenizer layers in **KerasNLP**:\n- [Preprocessing](https://keras.io/api/keras_nlp/preprocessing_layers/)\n- [Tokenizers](https://keras.io/api/keras_nlp/tokenizers/)","metadata":{"papermill":{"duration":0.019094,"end_time":"2023-11-17T02:52:47.340425","exception":false,"start_time":"2023-11-17T02:52:47.321331","status":"completed"},"tags":[]}},{"cell_type":"code","source":"preprocessor = keras_nlp.models.DebertaV3Preprocessor.from_preset(\n    preset=CFG.preset, # Name of the model\n    sequence_length=CFG.sequence_length, # Max sequence length, will be padded if shorter\n)","metadata":{"papermill":{"duration":0.918436,"end_time":"2023-11-17T02:52:48.278643","exception":false,"start_time":"2023-11-17T02:52:47.360207","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-28T05:17:23.340993Z","iopub.execute_input":"2023-11-28T05:17:23.341880Z","iopub.status.idle":"2023-11-28T05:17:24.291233Z","shell.execute_reply.started":"2023-11-28T05:17:23.341838Z","shell.execute_reply":"2023-11-28T05:17:24.290149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, let's examine what the output shape of the preprocessing layer looks like. The output shape of the layer can be represented as $(num\\_choices, sequence\\_length)$.","metadata":{"papermill":{"duration":0.019091,"end_time":"2023-11-17T02:52:48.317765","exception":false,"start_time":"2023-11-17T02:52:48.298674","status":"completed"},"tags":[]}},{"cell_type":"code","source":"inp = preprocessor(df.text.iloc[0])  # Process text for the first row\n\n# Display the shape of each processed output\nfor k, v in inp.items():\n    print(k, \":\", v.shape)","metadata":{"papermill":{"duration":0.229772,"end_time":"2023-11-17T02:52:48.567274","exception":false,"start_time":"2023-11-17T02:52:48.337502","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-28T05:17:25.045684Z","iopub.execute_input":"2023-11-28T05:17:25.047221Z","iopub.status.idle":"2023-11-28T05:17:25.218161Z","shell.execute_reply.started":"2023-11-28T05:17:25.047170Z","shell.execute_reply":"2023-11-28T05:17:25.217211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We'll use the `preprocessing_fn` function to transform each text option using the `dataset.map(preprocessing_fn)` method.","metadata":{"papermill":{"duration":0.019204,"end_time":"2023-11-17T02:52:48.606159","exception":false,"start_time":"2023-11-17T02:52:48.586955","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def preprocess_fn(text, label=None):\n    text = preprocessor(text)  # Preprocess text\n    return (text, label) if label is not None else text  # Return processed text and label if available","metadata":{"papermill":{"duration":0.026266,"end_time":"2023-11-17T02:52:48.651448","exception":false,"start_time":"2023-11-17T02:52:48.625182","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-28T05:17:26.778313Z","iopub.execute_input":"2023-11-28T05:17:26.778683Z","iopub.status.idle":"2023-11-28T05:17:26.784332Z","shell.execute_reply.started":"2023-11-28T05:17:26.778652Z","shell.execute_reply":"2023-11-28T05:17:26.783224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🍚 | DataLoader\n\nThe code below sets up a robust data flow pipeline using `tf.data.Dataset` for data processing. Notable aspects of `tf.data` include its ability to simplify pipeline construction and represent components in sequences.\n\nTo learn more about `tf.data`, refer to this [documentation](https://www.tensorflow.org/guide/data).","metadata":{"papermill":{"duration":0.018796,"end_time":"2023-11-17T02:52:48.689007","exception":false,"start_time":"2023-11-17T02:52:48.670211","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def build_dataset(texts, labels=None, batch_size=32,\n                  cache=False, drop_remainder=True,\n                  repeat=False, shuffle=1024):\n    AUTO = tf.data.AUTOTUNE  # AUTOTUNE option\n    slices = (texts,) if labels is None else (texts, labels)  # Create slices\n    ds = tf.data.Dataset.from_tensor_slices(slices)  # Create dataset from slices\n    ds = ds.cache() if cache else ds  # Cache dataset if enabled\n    ds = ds.map(preprocess_fn, num_parallel_calls=AUTO)  # Map preprocessing function\n    ds = ds.repeat() if repeat else ds  # Repeat dataset if enabled\n    opt = tf.data.Options()  # Create dataset options\n    if shuffle: \n        ds = ds.shuffle(shuffle, seed=CFG.seed)  # Shuffle dataset if enabled\n        opt.experimental_deterministic = False\n    ds = ds.with_options(opt)  # Set dataset options\n    ds = ds.batch(batch_size, drop_remainder=drop_remainder)  # Batch dataset\n    ds = ds.prefetch(AUTO)  # Prefetch next batch\n    return ds  # Return the built dataset","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.032673,"end_time":"2023-11-17T02:52:48.740580","exception":false,"start_time":"2023-11-17T02:52:48.707907","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-28T05:17:28.284318Z","iopub.execute_input":"2023-11-28T05:17:28.284691Z","iopub.status.idle":"2023-11-28T05:17:28.292838Z","shell.execute_reply.started":"2023-11-28T05:17:28.284661Z","shell.execute_reply":"2023-11-28T05:17:28.291660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fetch Train/Valid Dataset\n\nThe function below generates the training and validation datasets for a given fold.","metadata":{"papermill":{"duration":0.019243,"end_time":"2023-11-17T02:52:48.779902","exception":false,"start_time":"2023-11-17T02:52:48.760659","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_datasets(fold):\n    train_df = df[df.fold!=fold].sample(frac=1)  # Get training fold data\n        \n    train_texts = train_df.text.tolist()  # Extract training texts\n    train_labels = train_df.label.tolist()  # Extract training labels\n    \n    # Build training dataset\n    train_ds = build_dataset(train_texts, train_labels,\n                             batch_size=CFG.batch_size*CFG.replicas, cache=CFG.cache,\n                             shuffle=True, drop_remainder=True, repeat=True)\n\n    valid_df = df[df.fold==fold].sample(frac=1)  # Get validation fold data\n    valid_texts = valid_df.text.tolist()  # Extract validation texts\n    valid_labels = valid_df.label.tolist()  # Extract validation labels\n    \n    # Build validation dataset\n    valid_ds = build_dataset(valid_texts, valid_labels,\n                             batch_size=min(CFG.batch_size*CFG.replicas, len(valid_df)), cache=CFG.cache,\n                             shuffle=False, drop_remainder=True, repeat=False)\n    \n    return (train_ds, train_df), (valid_ds, valid_df)  # Return datasets and dataframes","metadata":{"_kg_hide-input":false,"papermill":{"duration":0.030683,"end_time":"2023-11-17T02:52:48.830431","exception":false,"start_time":"2023-11-17T02:52:48.799748","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-28T05:17:29.765987Z","iopub.execute_input":"2023-11-28T05:17:29.766409Z","iopub.status.idle":"2023-11-28T05:17:29.774334Z","shell.execute_reply.started":"2023-11-28T05:17:29.766365Z","shell.execute_reply":"2023-11-28T05:17:29.773296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🪄 | Wandb\n<img src=\"https://camo.githubusercontent.com/dd842f7b0be57140e68b2ab9cb007992acd131c48284eaf6b1aca758bfea358b/68747470733a2f2f692e696d6775722e636f6d2f52557469567a482e706e67\" width=\"400\" alt=\"Weights & Biases\" />\n\nTo monitor the training of my text-based model, I'll make use of **Weights & Biases**. Weights & Biases (W&B) is an MLOps platform that offers experiment tracking, dataset versioning, and model management functionalities, aiding in efficient model development. ","metadata":{"papermill":{"duration":0.019745,"end_time":"2023-11-17T02:52:48.871267","exception":false,"start_time":"2023-11-17T02:52:48.851522","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import wandb  # Import wandb library for experiment tracking\nimport wandb_addons # Additional wandb utilities\n\ntry:\n    from kaggle_secrets import UserSecretsClient  # Import UserSecretsClient\n    user_secrets = UserSecretsClient()  # Create secrets client instance\n    api_key = user_secrets.get_secret(\"WANDB\")  # Get API key from Kaggle secrets\n    wandb.login(key=api_key)  # Login to wandb with the API key\n    anonymous = None  # Set anonymous mode to None\nexcept:\n    anonymous = 'must'  # Set anonymous mode to 'must'\n    wandb.login(anonymous=anonymous, relogin=True)  # Login to wandb anonymously and relogin if needed","metadata":{"papermill":{"duration":2.255349,"end_time":"2023-11-17T02:52:51.146367","exception":false,"start_time":"2023-11-17T02:52:48.891018","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-28T05:17:30.960211Z","iopub.execute_input":"2023-11-28T05:17:30.960619Z","iopub.status.idle":"2023-11-28T05:17:34.478771Z","shell.execute_reply.started":"2023-11-28T05:17:30.960590Z","shell.execute_reply":"2023-11-28T05:17:34.477946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Logger\n\nThe following code cell contains code to log data to WandB. It is noteworthy that the newly released callbacks offer more flexibility in terms of customization, and they are more compact compared to the classic `WandbCallback`, making it easier to use. Here's a brief introduction to them:\n\n* **WandbModelCheckpoint**: This callback saves the model or weights using `tf.keras.callbacks.ModelCheckpoint`. Hence, we can harness the power of the official TensorFlow callback to log even `tf.keras.Model` subclass model in TPU.\n* **WandbMetricsLogger**: This callback simply logs all the metrics and losses.\n* **WandbEvalCallback**: This one is even more special. We can use it to log the model's prediction after a certain epoch/frequency. We can use it to save segmentation masks, bounding boxes, GradCAM within epochs to check intermediate results and so on.\n\nFor more details, please check the [official documentation](https://docs.wandb.ai/ref/python/integrations/keras).","metadata":{"papermill":{"duration":0.019771,"end_time":"2023-11-17T02:52:51.186477","exception":false,"start_time":"2023-11-17T02:52:51.166706","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# from wandb_addons.keras import WandbMetricsLogger\n# from wandb_addons.keras import WandbModelCheckpoint\n\n\n# Initializes the W&B run with a config file and W&B run settings.\ndef wandb_init(fold):\n    config = {k: v for k, v in dict(vars(CFG)).items() if '__' not in k}  # Create config dictionary\n    config.update({\"fold\": int(fold)})  # Add fold to config\n    run = wandb.init(project=\"llm-fake-text\",\n                     name=f\"fold-{fold}|max_seq-{CFG.sequence_length}|model-{CFG.preset}\",\n                     config=config,\n                     group=CFG.comment,\n                     save_code=True)\n    return run\n\n# Log best result for error analysis\ndef log_wandb():\n    wandb.log({'best_auc': best_auc, 'best_loss': best_loss, 'best_epoch': best_epoch})\n\n# Fetch W&B callbacks\ndef get_wb_callbacks(fold):\n#     wb_ckpt = WandbModelCheckpoint(f'fold{fold}.keras',\n#                                    monitor='val_auc',\n#                                    save_best_only=True,\n#                                    save_weights_only=False,\n#                                    mode='max')\n    wb_metr = wandb.keras.WandbMetricsLogger()\n    return [wb_metr]  # Return WandB callbacks\n","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.031884,"end_time":"2023-11-17T02:52:51.238529","exception":false,"start_time":"2023-11-17T02:52:51.206645","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-28T05:17:37.003401Z","iopub.execute_input":"2023-11-28T05:17:37.004117Z","iopub.status.idle":"2023-11-28T05:17:37.013435Z","shell.execute_reply.started":"2023-11-28T05:17:37.004085Z","shell.execute_reply":"2023-11-28T05:17:37.012418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ⚓ | LR Schedule\n\nImplementing a learning rate scheduler is crucial for transfer learning. The learning rate initiates at `lr_start` and gradually tapers down to `lr_min` using various techniques, including:\n- `step`: Lowering the learning rate in step-wise manner resembling stairs.\n- `cos`: Utilizing a cosine curve to gradually reduce the learning rate.\n- `exp`: Exponentially decreasing the learning rate.\n\n**Importance:** A well-structured learning rate schedule is essential for efficient model training, ensuring optimal convergence and avoiding issues such as overshooting or stagnation.","metadata":{"papermill":{"duration":0.019955,"end_time":"2023-11-17T02:52:51.278316","exception":false,"start_time":"2023-11-17T02:52:51.258361","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import math\n\ndef get_lr_callback(batch_size=8, mode='cos', epochs=10, plot=False):\n    lr_start, lr_max, lr_min = 0.6e-6, 0.5e-6 * batch_size, 0.3e-6\n    lr_ramp_ep, lr_sus_ep, lr_decay = 1, 0, 0.75\n\n    def lrfn(epoch):  # Learning rate update function\n        if epoch < lr_ramp_ep: lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n        elif epoch < lr_ramp_ep + lr_sus_ep: lr = lr_max\n        elif mode == 'exp': lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n        elif mode == 'step': lr = lr_max * lr_decay**((epoch - lr_ramp_ep - lr_sus_ep) // 2)\n        elif mode == 'cos':\n            decay_total_epochs, decay_epoch_index = epochs - lr_ramp_ep - lr_sus_ep + 3, epoch - lr_ramp_ep - lr_sus_ep\n            phase = math.pi * decay_epoch_index / decay_total_epochs\n            lr = (lr_max - lr_min) * 0.5 * (1 + math.cos(phase)) + lr_min\n        return lr\n\n    if plot:  # Plot lr curve if plot is True\n        plt.figure(figsize=(10, 5))\n        plt.plot(np.arange(epochs), [lrfn(epoch) for epoch in np.arange(epochs)], marker='o')\n        plt.xlabel('epoch'); plt.ylabel('lr')\n        plt.title('LR Scheduler')\n        plt.show()\n\n    return keras.callbacks.LearningRateScheduler(lrfn, verbose=False)  # Create lr callback","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.034052,"end_time":"2023-11-17T02:52:51.332316","exception":false,"start_time":"2023-11-17T02:52:51.298264","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-28T05:17:38.563547Z","iopub.execute_input":"2023-11-28T05:17:38.564464Z","iopub.status.idle":"2023-11-28T05:17:38.574495Z","shell.execute_reply.started":"2023-11-28T05:17:38.564427Z","shell.execute_reply":"2023-11-28T05:17:38.573563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_=get_lr_callback(CFG.batch_size*CFG.replicas, plot=True)","metadata":{"papermill":{"duration":0.310392,"end_time":"2023-11-17T02:52:51.662874","exception":false,"start_time":"2023-11-17T02:52:51.352482","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-28T05:17:39.700165Z","iopub.execute_input":"2023-11-28T05:17:39.700767Z","iopub.status.idle":"2023-11-28T05:17:39.990727Z","shell.execute_reply.started":"2023-11-28T05:17:39.700732Z","shell.execute_reply":"2023-11-28T05:17:39.989681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ☎️ | Callbacks\n\nThe function below will gather all the training callbacks, such as `lr_scheduler`, `model_checkpoint`, `wandb_logger`, and etc.","metadata":{"papermill":{"duration":0.021394,"end_time":"2023-11-17T02:52:51.705855","exception":false,"start_time":"2023-11-17T02:52:51.684461","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_callbacks(fold):\n    callbacks = []\n    lr_cb = get_lr_callback(CFG.batch_size*CFG.replicas)  # Get lr callback\n    ckpt_cb = keras.callbacks.ModelCheckpoint(f'fold{fold}.keras',\n                                              monitor='val_auc',\n                                              save_best_only=True,\n                                              save_weights_only=False,\n                                              mode='max')  # Get Model checkpoint callback\n    callbacks.extend([lr_cb, ckpt_cb])  # Add lr and checkpoint callbacks\n\n    if CFG.wandb:  # If WandB is enabled\n        wb_cbs = get_wb_callbacks(fold)  # Get WandB callbacks\n        callbacks.extend(wb_cbs) \n        \n    return callbacks  # Return the list of callbacks","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.029665,"end_time":"2023-11-17T02:52:51.756410","exception":false,"start_time":"2023-11-17T02:52:51.726745","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-28T05:17:41.244541Z","iopub.execute_input":"2023-11-28T05:17:41.245398Z","iopub.status.idle":"2023-11-28T05:17:41.251634Z","shell.execute_reply.started":"2023-11-28T05:17:41.245362Z","shell.execute_reply":"2023-11-28T05:17:41.250602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🤖 | Modeling\n\n","metadata":{"papermill":{"duration":0.019803,"end_time":"2023-11-17T02:52:51.796134","exception":false,"start_time":"2023-11-17T02:52:51.776331","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## KerasNLP Classifier\n\n<img src=\"https://keras.io/img/logo.png\" width=\"400\">\n\nThe `KerasNLP` library provides comprehensive, ready-to-use implementations of popular NLP model architectures. It features a variety of pre-trained models including `Bert`, `Roberta`, `DebertaV3`, and more. In this notebook, we'll showcase the usage of `DebertaV3`. However, feel free to explore all available models in the [KerasNLP documentation](https://keras.io/api/keras_nlp/models/). Also for a deeper understanding of `KerasNLP`, refer to the informative [getting started guide](https://keras.io/guides/keras_nlp/getting_started/).\n\nOur approach involves using `keras_nlp.models.XXClassifier` to process each text and generatie logits. These logits are passed through a `softmax` function to produce the final output.","metadata":{"papermill":{"duration":0.019767,"end_time":"2023-11-17T02:52:51.836114","exception":false,"start_time":"2023-11-17T02:52:51.816347","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def build_model():\n    # Create a DebertaV3Classifier model\n    classifier = keras_nlp.models.DebertaV3Classifier.from_preset(\n        CFG.preset,\n        preprocessor=None,\n        num_classes=1 # one output per one option, for five options total 5 outputs\n    )\n    inputs = classifier.input\n    logits = classifier(inputs)\n        \n    # Compute final output\n    outputs = keras.layers.Activation(\"sigmoid\")(logits)\n    model = keras.Model(inputs, outputs)\n    \n    # Compile the model with optimizer, loss, and metrics\n    model.compile(\n        optimizer=keras.optimizers.AdamW(5e-6),\n        loss=keras.losses.BinaryCrossentropy(label_smoothing=0.02),\n        metrics=[\n            keras.metrics.AUC(name=\"auc\"),\n        ],\n        jit_compile=True\n    )\n    return model","metadata":{"papermill":{"duration":0.030766,"end_time":"2023-11-17T02:52:51.887396","exception":false,"start_time":"2023-11-17T02:52:51.856630","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-28T05:17:43.325649Z","iopub.execute_input":"2023-11-28T05:17:43.326027Z","iopub.status.idle":"2023-11-28T05:17:43.333030Z","shell.execute_reply.started":"2023-11-28T05:17:43.325979Z","shell.execute_reply":"2023-11-28T05:17:43.332053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# with strategy.scope\nmodel = build_model()","metadata":{"papermill":{"duration":9.108263,"end_time":"2023-11-17T02:53:01.015916","exception":false,"start_time":"2023-11-17T02:52:51.907653","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-28T05:17:44.059806Z","iopub.execute_input":"2023-11-28T05:17:44.060458Z","iopub.status.idle":"2023-11-28T05:17:50.909800Z","shell.execute_reply.started":"2023-11-28T05:17:44.060421Z","shell.execute_reply":"2023-11-28T05:17:50.908908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model Summary","metadata":{"papermill":{"duration":0.031844,"end_time":"2023-11-17T02:53:01.077787","exception":false,"start_time":"2023-11-17T02:53:01.045943","status":"completed"},"tags":[]}},{"cell_type":"code","source":"model.summary()","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.074499,"end_time":"2023-11-17T02:53:01.183743","exception":false,"start_time":"2023-11-17T02:53:01.109244","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-28T05:17:52.446134Z","iopub.execute_input":"2023-11-28T05:17:52.446947Z","iopub.status.idle":"2023-11-28T05:17:52.472568Z","shell.execute_reply.started":"2023-11-28T05:17:52.446916Z","shell.execute_reply":"2023-11-28T05:17:52.471703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model Plot","metadata":{"papermill":{"duration":0.030366,"end_time":"2023-11-17T02:53:01.245233","exception":false,"start_time":"2023-11-17T02:53:01.214867","status":"completed"},"tags":[]}},{"cell_type":"code","source":"keras.utils.plot_model(model, show_shapes=True)","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.558424,"end_time":"2023-11-17T02:53:01.832860","exception":false,"start_time":"2023-11-17T02:53:01.274436","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-28T05:17:54.131610Z","iopub.execute_input":"2023-11-28T05:17:54.131985Z","iopub.status.idle":"2023-11-28T05:17:54.448322Z","shell.execute_reply.started":"2023-11-28T05:17:54.131954Z","shell.execute_reply":"2023-11-28T05:17:54.447281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🚂 | Training","metadata":{"papermill":{"duration":0.031234,"end_time":"2023-11-17T02:53:01.895856","exception":false,"start_time":"2023-11-17T02:53:01.864622","status":"completed"},"tags":[]}},{"cell_type":"code","source":"for fold in CFG.selected_folds:\n    # Initialize Weights and Biases if enabled\n    if CFG.wandb:\n        run = wandb_init(fold)\n\n    # Get train and validation datasets\n    (train_ds, train_df), (valid_ds, valid_df) = get_datasets(fold)\n    \n    # Get callback functions for training\n    callbacks = get_callbacks(fold)\n\n    # Print training information\n    print('#' * 50)\n    print(f'\\tFold: {fold + 1} | Model: {CFG.preset}\\n\\tBatch Size: {CFG.batch_size * CFG.replicas} | Scheduler: {CFG.scheduler}')\n    print(f'\\tNum Train: {len(train_df)} | Num Valid: {len(valid_df)}')\n    print('#' * 50)\n    \n    # Clear TensorFlow session and build the model within the strategy scope\n    K.clear_session()\n    with strategy.scope():\n        model = build_model()\n\n    # Start training the model\n    history = model.fit(\n        train_ds,\n        epochs=CFG.epochs,\n        validation_data=valid_ds,\n        callbacks=callbacks,\n        steps_per_epoch=int(len(train_df) / CFG.batch_size / CFG.replicas),\n    )\n    \n    # Find the epoch with the best validation accuracy\n    best_epoch = np.argmax(model.history.history['val_auc'])\n    best_auc = model.history.history['val_auc'][best_epoch]\n    best_loss = model.history.history['val_loss'][best_epoch]\n\n    # Print and display best results\n    print(f'\\n{\"=\" * 17} FOLD {fold} RESULTS {\"=\" * 17}')\n    print(f'>>>> BEST Loss  : {best_loss:.3f}\\n>>>> BEST AUC   : {best_auc:.3f}\\n>>>> BEST Epoch : {best_epoch}')\n    print('=' * 50)\n    \n    # Log best result on Weights and Biases (wandb) if enabled\n    if CFG.wandb:\n        log_wandb()  # Log results\n        wandb.run.finish()  # Finish the run\n#         display(ipd.IFrame(run.url, width=1080, height=720)) # show wandb dashboard\n    print(\"\\n\\n\")","metadata":{"_kg_hide-input":true,"papermill":{"duration":16705.709123,"end_time":"2023-11-17T07:31:27.636517","exception":false,"start_time":"2023-11-17T02:53:01.927394","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-28T05:17:56.509520Z","iopub.execute_input":"2023-11-28T05:17:56.509868Z","iopub.status.idle":"2023-11-28T09:59:35.205560Z","shell.execute_reply.started":"2023-11-28T05:17:56.509839Z","shell.execute_reply":"2023-11-28T09:59:35.204559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🧪 | Prediction","metadata":{"papermill":{"duration":3.041231,"end_time":"2023-11-17T07:31:39.739358","exception":false,"start_time":"2023-11-17T07:31:36.698127","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Make predictions using the trained model on last validation data\npredictions = model.predict(\n    \n    valid_ds,\n    batch_size=min(CFG.batch_size * CFG.replicas * 2, len(valid_df)), # max batch size = valid size\n    verbose=1\n)","metadata":{"papermill":{"duration":130.354862,"end_time":"2023-11-17T07:33:53.099541","exception":false,"start_time":"2023-11-17T07:31:42.744679","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-28T10:10:46.127751Z","iopub.execute_input":"2023-11-28T10:10:46.128161Z","iopub.status.idle":"2023-11-28T10:12:50.614625Z","shell.execute_reply.started":"2023-11-28T10:10:46.128131Z","shell.execute_reply":"2023-11-28T10:12:50.613803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Format predictions and true answers\npred_answers = (predictions > 0.5).astype(int).squeeze()\ntrue_answers = valid_df.label.values\n\n# Check 5 Predictions\nprint(\"# Predictions\\n\")\nfor i in range(5):\n    row = valid_df.iloc[i]\n    text  = row.text\n    pred_answer = CFG.label2name[pred_answers[i]]\n    true_answer = CFG.label2name[true_answers[i]]\n    print(f\"❓ Text {i+1}:\\n{text[:100]} .... {text[-100:]}\\n\")\n    print(f\"✅ True: {true_answer}\\n\")\n    print(f\"🤖 Predicted: {pred_answer}\\n\")\n    print(\"-\"*90, \"\\n\")","metadata":{"_kg_hide-input":true,"papermill":{"duration":3.178633,"end_time":"2023-11-17T07:33:59.400551","exception":false,"start_time":"2023-11-17T07:33:56.221918","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-28T10:13:00.088763Z","iopub.execute_input":"2023-11-28T10:13:00.089175Z","iopub.status.idle":"2023-11-28T10:13:00.098732Z","shell.execute_reply.started":"2023-11-28T10:13:00.089142Z","shell.execute_reply":"2023-11-28T10:13:00.097594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -r /kaggle/working/wandb  ","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"papermill":{"duration":4.161198,"end_time":"2023-11-17T07:34:13.120709","exception":false,"start_time":"2023-11-17T07:34:08.959511","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-28T10:13:01.280806Z","iopub.execute_input":"2023-11-28T10:13:01.281186Z","iopub.status.idle":"2023-11-28T10:13:02.367202Z","shell.execute_reply.started":"2023-11-28T10:13:01.281157Z","shell.execute_reply":"2023-11-28T10:13:02.365775Z"},"trusted":true},"execution_count":null,"outputs":[]}]}