step4_claim_label_cls.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "103cd24f-29d7-401e-861a-26a47851796e",
   "metadata": {},
   "source": [
    "# Step 4: Claim label Classification"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5a908b53-9456-47c5-87ef-4696d3918057",
   "metadata": {},
   "source": [
    "# Readme\n",
    "*This notebook responsible for classify the claim label base on the evidence we found on previous stages*"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5646ef7-0f7f-4cbe-bd2e-aba29030a0c7",
   "metadata": {},
   "source": [
    "# 1.DataSet Processing\n",
    "(You can add as many code blocks and text blocks as you need. However, YOU SHOULD NOT MODIFY the section title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "025e488b-986f-4d2c-9c5a-8597f77e1561",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.layers import BatchNormalization, TextVectorization, Embedding, Bidirectional, LSTM, Conv1D, GlobalMaxPooling1D, Dense, Dropout, Lambda, Dot, Reshape, GlobalAveragePooling1D, Flatten\n",
    "from tensorflow.keras import Model, Input, optimizers, layers, metrics\n",
    "from sklearn.model_selection import train_test_split\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint, Callback\n",
    "from tensorflow.keras import backend as K\n",
    "\n",
    "import spacy\n",
    "import string\n",
    "import time\n",
    "from collections import Counter\n",
    "import string\n",
    "import ast"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4ed35498-7e9c-412b-be71-809be234d968",
   "metadata": {},
   "source": [
    "Load the result from step 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "d1511641-94b3-49de-a8b1-2ef90679f752",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv(\"data_folder/train_retrival_result_with_transformer.csv\")\n",
    "evidence = pd.read_csv(\"data_folder/evidence.csv\")\n",
    "dev = pd.read_csv(\"data_folder/dev_retrival_result_with_transformer.csv\")\n",
    "test = pd.read_csv(\"data_folder/test_retrival_result_with_transformer.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "b05f4c1a-9f8b-4639-83b3-e63f9f2f79f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "dev['predicted_add_evidence_transformer'] = dev['predicted_add_evidence_transformer'].apply(lambda x: ast.literal_eval(x))\n",
    "train['predicted_add_evidence_transformer'] = train['predicted_add_evidence_transformer'].apply(lambda x: ast.literal_eval(x))\n",
    "test['predicted_add_evidence_transformer'] = test['predicted_add_evidence_transformer'].apply(lambda x: ast.literal_eval(x))\n",
    "dev['predicted_rm_evidence_transformer'] = dev['predicted_rm_evidence_transformer'].apply(lambda x: ast.literal_eval(x))\n",
    "train['predicted_rm_evidence_transformer'] = train['predicted_rm_evidence_transformer'].apply(lambda x: ast.literal_eval(x))\n",
    "test['predicted_rm_evidence_transformer'] = test['predicted_rm_evidence_transformer'].apply(lambda x: ast.literal_eval(x))\n",
    "dev['predict_evidence_bm25'] = dev['predict_evidence_bm25'].apply(lambda x: ast.literal_eval(x))\n",
    "train['predict_evidence_bm25'] = train['predict_evidence_bm25'].apply(lambda x: ast.literal_eval(x))\n",
    "test['predict_evidence_bm25'] = test['predict_evidence_bm25'].apply(lambda x: ast.literal_eval(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c59af075-e031-49ea-a675-01c619d2ec29",
   "metadata": {},
   "source": [
    "Create our final predicted evidence column by kept top 4 evidences found by BM25 and adding first top evidences found by transformer if its not found by BM25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "0a6bf834-1513-4b5f-8d4d-867c54030025",
   "metadata": {},
   "outputs": [],
   "source": [
    "def combine_evidences(row):\n",
    "    # Get the first element from the predicted_add_evidence_transformer list\n",
    "    if row['predicted_add_evidence_transformer']:\n",
    "        first_add_evidence = row['predicted_add_evidence_transformer'][0]\n",
    "    else:\n",
    "        first_add_evidence = None\n",
    "    # Get the BM25 predicted evidences and initialize a new list if it's None\n",
    "    bm25_evidences = row['predict_evidence_bm25'][:3] if row['predict_evidence_bm25'] is not None else []\n",
    "    \n",
    "    # Check if the first element from predicted_add_evidence_transformer is not in predict_evidence_bm25\n",
    "    if first_add_evidence is not None and first_add_evidence not in bm25_evidences:\n",
    "        bm25_evidences.append(first_add_evidence)\n",
    "    \n",
    "    return bm25_evidences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "26257e72-d6e4-4de6-a2ed-8abf27d1fca4",
   "metadata": {},
   "outputs": [],
   "source": [
    "train['predicted_evidence'] = train.apply(combine_evidences, axis=1)\n",
    "dev['predicted_evidence'] = dev.apply(combine_evidences, axis=1)\n",
    "test['predicted_evidence'] = test.apply(combine_evidences, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "32d296c3-5ee7-42aa-86d5-ed290a2e770d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average Evidence Recall    = 0.18333333333333335\n",
      "Average Evidence Precision = 0.1412337662337662\n",
      "Average Evidence F-Score   = 0.147979797979798\n",
      "Average Evidence Recall    = 0.4887757871878393\n",
      "Average Evidence Precision = 0.3319082519001086\n",
      "Average Evidence F-Score   = 0.363018975233959\n"
     ]
    }
   ],
   "source": [
    "def print_retrival_result(df):\n",
    "    evidence_recall_scores = []\n",
    "    evidence_precision_scores = []\n",
    "    evidence_fscore_scores = []\n",
    "    \n",
    "    # Iterate over each row in the dev DataFrame to evaluate evidence retrieval\n",
    "    for index, row in df.iterrows():\n",
    "        true_evidences = set(ast.literal_eval(row['evidences']))    \n",
    "        predicted_evidences = set(row['predicted_evidence'])\n",
    "        # Initialize counters for correct predictions\n",
    "        evidence_correct = len(true_evidences & predicted_evidences)\n",
    "        \n",
    "        # Calculate recall, precision, and F-score\n",
    "        if len(true_evidences) > 0 and len(predicted_evidences) > 0:\n",
    "            evidence_recall = evidence_correct / len(true_evidences)\n",
    "            evidence_precision = evidence_correct / len(predicted_evidences)\n",
    "            if evidence_recall + evidence_precision > 0:\n",
    "                evidence_fscore = (2 * evidence_precision * evidence_recall) / (evidence_precision + evidence_recall)\n",
    "            else:\n",
    "                evidence_fscore = 0.0\n",
    "        else:\n",
    "            evidence_recall = 0.0\n",
    "            evidence_precision = 0.0\n",
    "            evidence_fscore = 0.0\n",
    "    \n",
    "        # Store the scores\n",
    "        evidence_recall_scores.append(evidence_recall)\n",
    "        evidence_precision_scores.append(evidence_precision)\n",
    "        evidence_fscore_scores.append(evidence_fscore)\n",
    "        \n",
    "    # Calculate mean scores across all instances\n",
    "    mean_recall = np.mean(evidence_recall_scores)\n",
    "    mean_precision = np.mean(evidence_precision_scores)\n",
    "    mean_fscore = np.mean(evidence_fscore_scores)\n",
    "    \n",
    "    # Output the aggregate performance\n",
    "    print(f\"Average Evidence Recall    = {mean_recall}\")\n",
    "    print(f\"Average Evidence Precision = {mean_precision}\")\n",
    "    print(f\"Average Evidence F-Score   = {mean_fscore}\")\n",
    "print_retrival_result(dev)\n",
    "print_retrival_result(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "45f90599-df34-4380-be34-945db7fa2938",
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "def remove_punctuation(input_text):\n",
    "    # Lowercase the input text to standardize it\n",
    "    input_text = input_text.lower()\n",
    "    \n",
    "    # Remove punctuation using a translation table\n",
    "    translator = str.maketrans('', '', string.punctuation)\n",
    "    input_text = input_text.translate(translator)\n",
    "    \n",
    "    # Tokenize the text into words by splitting on whitespace\n",
    "    tokens = input_text.split()\n",
    "    \n",
    "    # Join words back into one string and return\n",
    "    return ' '.join(tokens)\n",
    "\n",
    "# Apply the function to your dataframes\n",
    "evidence['evidence_text'] = evidence['evidence_text'].apply(remove_punctuation)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e75f7d32-a281-45cd-a98b-2ccd3c588296",
   "metadata": {},
   "source": [
    "Concat the claim text with all of its predicted evidences"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ca0551d-d321-444a-b4a9-a920b602ee74",
   "metadata": {},
   "source": [
    "The parameter include_all_ground_truth_ev indicate that whether the training process will be trained on true evidence or predicted evidence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "1299e150-2979-4ec5-bff3-8b499bafa60e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def concat_text(row, include_all_ground_truth_ev = True):\n",
    "    concatenated_text = \"<CLS> \" + row['claim_text']\n",
    "    for ev_index in ast.literal_eval(row['evidences']) if include_all_ground_truth_ev else row['predicted_evidence']:\n",
    "        concatenated_text += \" <SEP> \" + evidence.iloc[ev_index].evidence_text\n",
    "    return concatenated_text"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a3083117-a83f-4444-8b4e-820a5e59f164",
   "metadata": {},
   "source": [
    "The result of the model predicted on the ground truth evidence could be reproduce if change the following parameter to true"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "6774a72a-cac0-4fff-bae8-bf703a274fe1",
   "metadata": {},
   "outputs": [],
   "source": [
    "train[\"concatenated_text\"] = train.apply(lambda x: concat_text(x, False), axis=1)\n",
    "dev[\"concatenated_text\"] = dev.apply(lambda x:concat_text(x, False), axis=1)\n",
    "test[\"concatenated_text\"] = test.apply(lambda x:concat_text(x, False), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "84389daf-e6ba-4e15-b4f2-0df7c3715431",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start processing!\n",
      "Vocabulary size on all claim text: 12145\n",
      "Processed and encoded data.\n"
     ]
    }
   ],
   "source": [
    "# Define file paths\n",
    "sequence_length = 256\n",
    "print(\"Start processing!\")\n",
    "# Concatenate priority texts\n",
    "priority_texts = pd.concat([train['concatenated_text'], dev['concatenated_text'], test['concatenated_text']])\n",
    "\n",
    "# Create the TextVectorization layer\n",
    "vectorize_layer = tf.keras.layers.TextVectorization(\n",
    "    output_mode=\"int\",\n",
    "    output_sequence_length=sequence_length\n",
    ")\n",
    "\n",
    "# Adapt the vectorization layer on priority texts first\n",
    "vectorize_layer.adapt(priority_texts)\n",
    "print(\"Vocabulary size on all claim text:\", len(vectorize_layer.get_vocabulary()))\n",
    "max_features = len(vectorize_layer.get_vocabulary())\n",
    "# Encode texts\n",
    "train_encoded = vectorize_layer(train[\"concatenated_text\"].to_numpy())\n",
    "dev_encoded = vectorize_layer(dev[\"concatenated_text\"].to_numpy())\n",
    "test_encoded = vectorize_layer(test[\"concatenated_text\"].to_numpy())\n",
    "print(\"Processed and encoded data.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "a49437e4-6c8e-455d-852b-4511539d2513",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'DISPUTED': 0, 'NOT_ENOUGH_INFO': 1, 'REFUTES': 2, 'SUPPORTS': 3}\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "# Assuming train_for_multitask is your DataFrame\n",
    "label_encoder = LabelEncoder()\n",
    "\n",
    "# Convert categorical claim labels to integers\n",
    "train_label = label_encoder.fit_transform(train['claim_label'])\n",
    "dev_label = label_encoder.fit_transform(dev['claim_label'])\n",
    "\n",
    "# Display unique classes and their mapping to check\n",
    "print(dict(zip(label_encoder.classes_, label_encoder.transform(label_encoder.classes_))))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0b775e61-422d-4e9f-aad0-a3c4ba69b31c",
   "metadata": {},
   "source": [
    "# 2. Model Implementation\n",
    "(You can add as many code blocks and text blocks as you need. However, YOU SHOULD NOT MODIFY the section title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "16c49118-34dd-421e-b471-3dda44dbcdc8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 256)]             0         \n",
      "_________________________________________________________________\n",
      "embedding_2 (Embedding)      (None, 256, 1024)         12436480  \n",
      "_________________________________________________________________\n",
      "tf.__operators__.add_1 (TFOp (None, 256, 1024)         0         \n",
      "_________________________________________________________________\n",
      "transformer_block_2 (Transfo (None, 256, 1024)         17057920  \n",
      "_________________________________________________________________\n",
      "transformer_block_3 (Transfo (None, 256, 1024)         17057920  \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_1 (Glob (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "dropout_9 (Dropout)          (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "claim_label (Dense)          (None, 4)                 4100      \n",
      "=================================================================\n",
      "Total params: 46,556,420\n",
      "Trainable params: 46,556,420\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "class TransformerBlock(tf.keras.layers.Layer):\n",
    "    def __init__(self, embed_dim, num_heads, ff_dim, rate=0.1, **kwargs):\n",
    "        super(TransformerBlock, self).__init__(**kwargs)  # Pass any extra arguments to the superclass\n",
    "        self.embed_dim = embed_dim\n",
    "        self.num_heads = num_heads\n",
    "        self.ff_dim = ff_dim\n",
    "        self.rate = rate\n",
    "        self.att = tf.keras.layers.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\n",
    "        self.ffn = tf.keras.Sequential([\n",
    "            tf.keras.layers.Dense(ff_dim, activation=\"relu\"),\n",
    "            tf.keras.layers.Dense(embed_dim),\n",
    "        ])\n",
    "        self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.dropout1 = tf.keras.layers.Dropout(rate)\n",
    "        self.dropout2 = tf.keras.layers.Dropout(rate)\n",
    "\n",
    "    def call(self, inputs, training=False):\n",
    "        attn_output = self.att(inputs, inputs)\n",
    "        attn_output = self.dropout1(attn_output, training=training)\n",
    "        out1 = self.layernorm1(inputs + attn_output)\n",
    "        ffn_output = self.ffn(out1)\n",
    "        ffn_output = self.dropout2(ffn_output, training=training)\n",
    "        return self.layernorm2(out1 + ffn_output)\n",
    "\n",
    "    def get_config(self):\n",
    "        config = super(TransformerBlock, self).get_config()\n",
    "        config.update({\n",
    "            \"embed_dim\": self.embed_dim,\n",
    "            \"num_heads\": self.num_heads,\n",
    "            \"ff_dim\": self.ff_dim,\n",
    "            \"rate\": self.rate\n",
    "        })\n",
    "        return config\n",
    "\n",
    "# embedding model with lstm approach     \n",
    "def create_cls_model(sequence_length, max_features, num_transformer_blocks, embedding_dim, num_heads, ff_dim, rate):\n",
    "    input_layer = Input(shape=(sequence_length,), dtype=\"int64\")\n",
    "    \n",
    "    # Embedding layer with positional encoding\n",
    "    embedding_layer = Embedding(max_features, embedding_dim)\n",
    "    x = embedding_layer(input_layer)\n",
    "    # Adding positional encoding\n",
    "    \n",
    "    position_embedding = Embedding(input_dim=sequence_length, output_dim=embedding_dim)\n",
    "    positions = tf.range(start=0, limit=sequence_length, delta=1)\n",
    "    x += position_embedding(positions)\n",
    "    \n",
    "    # Transformer blocks\n",
    "    for _ in range(num_transformer_blocks):\n",
    "        x = TransformerBlock(embedding_dim, num_heads, ff_dim, rate)(x)\n",
    "    x = GlobalMaxPooling1D()(x)\n",
    "    x = Dropout(0.1)(x)\n",
    "    # Dense layers and output\n",
    "    claim_output = Dense(4, activation='softmax', name=\"claim_label\")(x)\n",
    "    \n",
    "    # Compile model\n",
    "    model = Model(inputs=input_layer, outputs=claim_output)\n",
    "    return model\n",
    "\n",
    "# Initialize and compile the model as before\n",
    "claim_cls = create_cls_model(sequence_length=sequence_length, max_features=max_features, num_transformer_blocks = 2,\n",
    "                                         embedding_dim = 1024, num_heads=4, ff_dim=128, rate=0.1)\n",
    "claim_cls.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "4e533cc7-3c5f-4521-b955-2561777161c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "claim_cls.compile(\n",
    "    optimizer=optimizers.Adam(1e-5),\n",
    "    loss={\n",
    "        'claim_label': 'sparse_categorical_crossentropy'\n",
    "    },\n",
    "    metrics={\n",
    "        'claim_label': ['accuracy']\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "7fae0e3e-da0e-47e2-8f16-4c1e66064c69",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "39/39 [==============================] - 4s 90ms/step - loss: 1.2599 - accuracy: 0.4707 - val_loss: 1.2986 - val_accuracy: 0.4286\n",
      "Epoch 2/10\n",
      "39/39 [==============================] - 3s 90ms/step - loss: 1.2370 - accuracy: 0.4731 - val_loss: 1.3045 - val_accuracy: 0.4481\n",
      "Epoch 3/10\n",
      "39/39 [==============================] - 3s 90ms/step - loss: 1.2097 - accuracy: 0.4935 - val_loss: 1.3117 - val_accuracy: 0.4481\n",
      "Epoch 4/10\n",
      "39/39 [==============================] - 3s 90ms/step - loss: 1.1646 - accuracy: 0.5098 - val_loss: 1.3522 - val_accuracy: 0.4545\n",
      "Epoch 5/10\n",
      "39/39 [==============================] - 3s 90ms/step - loss: 1.1576 - accuracy: 0.5171 - val_loss: 1.2968 - val_accuracy: 0.4286\n",
      "Epoch 6/10\n",
      "39/39 [==============================] - 3s 90ms/step - loss: 1.1067 - accuracy: 0.5350 - val_loss: 1.2987 - val_accuracy: 0.4091\n",
      "Epoch 7/10\n",
      "39/39 [==============================] - 3s 90ms/step - loss: 1.0571 - accuracy: 0.5423 - val_loss: 1.2996 - val_accuracy: 0.4026\n",
      "Epoch 8/10\n",
      "39/39 [==============================] - 3s 90ms/step - loss: 1.0106 - accuracy: 0.5806 - val_loss: 1.3037 - val_accuracy: 0.3961\n",
      "Epoch 9/10\n",
      "39/39 [==============================] - 3s 90ms/step - loss: 0.9588 - accuracy: 0.6091 - val_loss: 1.2965 - val_accuracy: 0.4026\n",
      "Epoch 10/10\n",
      "39/39 [==============================] - 3s 90ms/step - loss: 0.9488 - accuracy: 0.6116 - val_loss: 1.3283 - val_accuracy: 0.4481\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1b40737a670>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "claim_cls.fit(train_encoded, train_label, epochs=10, validation_data=(dev_encoded, dev_label))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1c2e0473-1cf0-488e-ae9e-1ded353730a9",
   "metadata": {},
   "source": [
    "# 3.Testing and Evaluation\n",
    "(You can add as many code blocks and text blocks as you need. However, YOU SHOULD NOT MODIFY the section title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "4ea77756-9491-4936-9921-cdcfcb8731e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = claim_cls.predict(test_encoded)\n",
    "predicted_labels_indices = np.argmax(predictions, axis=1)\n",
    "\n",
    "# Decode predictions back to original labels\n",
    "predicted_labels = label_encoder.inverse_transform(predicted_labels_indices)\n",
    "\n",
    "# Attach predictions to the DataFrame\n",
    "test['claim_label'] = predicted_labels"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53fc8fb7-e1a3-4e77-881c-e2ef1821223a",
   "metadata": {},
   "source": [
    "You might need to change the data path of test json file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "4f73f097-1cb7-488d-9bae-5257578ff94f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "input_json_path = '../project-data/test-claims-unlabelled.json'\n",
    "with open(input_json_path, 'r') as file:\n",
    "    data = json.load(file)\n",
    "\n",
    "ordered_json_keys = list(data.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "d55a5c90-3bd4-48cc-b6e7-d785b4c696e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "for idx, claim_id in enumerate(ordered_json_keys):\n",
    "    row = test.iloc[idx]\n",
    "    evidences_formatted = [f\"evidence-{ev}\" for ev in row['predicted_evidence']]\n",
    "    data[claim_id]['claim_label'] = row['claim_label']\n",
    "    data[claim_id]['evidences'] = evidences_formatted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "f393e16f-3122-45f4-88e5-a199135e5d47",
   "metadata": {},
   "outputs": [],
   "source": [
    "output_json_path = 'test-output.json'\n",
    "with open(output_json_path, 'w') as file:\n",
    "    json.dump(data, file, indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7c56ebc9-ce37-4efe-8515-0b68528114b2",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}