platypus_person.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c1dabf49-55c5-47c1-961c-c8e02b712eb0",
   "metadata": {},
   "source": [
    "# Platypus 2 w/ person"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c57ce2cb-fc85-4fc2-9708-8fa387faa645",
   "metadata": {},
   "source": [
    "## 7B - Context window 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6b2ac437-9309-45a1-b2ed-7643169863fc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-12-12 16:38:07.774487: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ed9dba81880344f89f8bf6a9dd63b262",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.8/dist-packages/transformers/generation/configuration_utils.py:362: UserWarning: `do_sample` is set to `False`. However, `temperature` is set to `0.001` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.8/dist-packages/sklearn/metrics/_classification.py:1469: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/usr/local/lib/python3.8/dist-packages/sklearn/metrics/_classification.py:1469: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>F1</th>\n",
       "      <th>Recall</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.257109</td>\n",
       "      <td>0.190476</td>\n",
       "      <td>0.506378</td>\n",
       "      <td>0.190476</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         F1    Recall  Precision  Accuracy\n",
       "0  0.257109  0.190476   0.506378  0.190476"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "os.environ['TRANSFORMERS_CACHE'] = \"cache_dir\"\n",
    "import transformers\n",
    "from transformers import pipeline\n",
    "from transformers import BitsAndBytesConfig\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "import re\n",
    "import pandas as pd\n",
    "import torch\n",
    "import langchain\n",
    "from langchain.llms import HuggingFacePipeline\n",
    "from langchain.prompts import PromptTemplate\n",
    "from langchain.chains import LLMChain\n",
    "import sklearn\n",
    "import time\n",
    "from sacrebleu import BLEU\n",
    "import tqdm.contrib\n",
    "df = pd.read_csv(\"./data/KnowledgeTransfers-one_sent.csv\", sep=\",\")\n",
    "quantization_config = BitsAndBytesConfig(\n",
    "    load_in_4bit=True,\n",
    "    bnb_4bit_compute_dtype=torch.bfloat16\n",
    ")\n",
    "\n",
    "model_id = \"garage-bAInd/Platypus2-7B\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=True)\n",
    "model_bf16 = AutoModelForCausalLM.from_pretrained(model_id, device_map=\"auto\", quantization_config=quantization_config)\n",
    "def run_llm(snippets, persons1, persons2):\n",
    "    outputs = []\n",
    "    for snippet, person1, person2 in zip(snippets, persons1, persons2):\n",
    "        pipe = pipeline(\n",
    "        \"text-generation\", model=model_bf16, tokenizer=tokenizer, max_new_tokens=200, temperature = 0.001\n",
    "        )\n",
    "        hf = HuggingFacePipeline(pipeline=pipe)\n",
    "        template = \"\"\"\n",
    "        ####\n",
    "        Instruction: You are a literary scholar.\n",
    "        What is the family relation between {person1} and {person2} in the German text {drama_snippet}?\n",
    "        The possible family relations are parent, child, uncle, siblings, cousins.\n",
    "        Answer in a single sentence in the following format: The family relation between {person1} and {person2} is >>correct family relation<<.\n",
    "        ####\n",
    "        \"\"\"\n",
    "        drama_snippet = str(snippet)\n",
    "        person1 = str(person1)\n",
    "        person2 = str(person2)\n",
    "        prompt = PromptTemplate(\n",
    "            input_variables=[\"drama_snippet\", \"person1\", \"person2\"],\n",
    "            template=template,\n",
    "        )\n",
    "        chain = LLMChain(llm=hf, prompt=prompt)\n",
    "        output = chain.run({\"drama_snippet\": {drama_snippet}, \"person1\": person1, \"person2\": person2})\n",
    "        outputs.append(output)\n",
    "    return outputs\n",
    "df_sample = df.loc[(df.annotation != \"no-transfer\") & (df.relation.isin([\"parent_of\", \"child_of\", \"siblings\", \"uncle_of\", \"cousins\"]))]\n",
    "llm_outputs = run_llm(df_sample.context, df_sample.arg1, df_sample.arg2)\n",
    "predictions_relation = []\n",
    "for o in llm_outputs:\n",
    "    if \"parent\" in o:\n",
    "        prediction_relation = [\"parent_of\"]\n",
    "    elif \"child\" in o:\n",
    "        prediction_relation = [\"child_of\"]\n",
    "    elif \"siblings\" in o:\n",
    "        prediction_relation = [\"siblings\"]\n",
    "    elif \"uncle\" in o:\n",
    "        prediction_relation = [\"uncle_of\"]\n",
    "    elif \"cousins\" in o:\n",
    "        prediction_relation = [\"cousins\"]\n",
    "    else:\n",
    "        prediction_relation = [\"else\"]\n",
    "    predictions_relation.extend(prediction_relation)\n",
    "results_relation = pd.DataFrame(list(zip(list(df_sample.relation), predictions_relation)), columns = [\"key\", \"pred\"])\n",
    "pd.DataFrame(list(zip([sklearn.metrics.f1_score(results_relation.key, results_relation.pred, average=\"weighted\")], \n",
    "             [sklearn.metrics.recall_score(results_relation.key, results_relation.pred, average=\"weighted\")], \n",
    "             [sklearn.metrics.precision_score(results_relation.key, results_relation.pred, average=\"weighted\")],\n",
    "             [sklearn.metrics.accuracy_score(results_relation.key, results_relation.pred)])), \n",
    "             columns = [\"F1\", \"Recall\", \"Precision\", \"Accuracy\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1d179b6-6963-4c05-bf1e-4f716e54c5a7",
   "metadata": {},
   "source": [
    "## 7B - Context window 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ad0c52a5-09a6-491e-ab5b-4ad095a6aa79",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0b2400388b7147109b8e72d627cab64e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.8/dist-packages/transformers/generation/configuration_utils.py:362: UserWarning: `do_sample` is set to `False`. However, `temperature` is set to `0.001` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.8/dist-packages/sklearn/metrics/_classification.py:1469: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>F1</th>\n",
       "      <th>Recall</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.293283</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.309524</td>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         F1    Recall  Precision  Accuracy\n",
       "0  0.293283  0.333333   0.309524  0.333333"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "os.environ['TRANSFORMERS_CACHE'] = \"cache_dir\"\n",
    "import transformers\n",
    "from transformers import pipeline\n",
    "from transformers import BitsAndBytesConfig\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "import re\n",
    "import pandas as pd\n",
    "import torch\n",
    "import langchain\n",
    "from langchain.llms import HuggingFacePipeline\n",
    "from langchain.prompts import PromptTemplate\n",
    "from langchain.chains import LLMChain\n",
    "import sklearn\n",
    "import time\n",
    "from sacrebleu import BLEU\n",
    "import tqdm.contrib\n",
    "df = pd.read_csv(\"./data/KnowledgeTransfers-two_sent.csv\", sep=\",\")\n",
    "quantization_config = BitsAndBytesConfig(\n",
    "    load_in_4bit=True,\n",
    "    bnb_4bit_compute_dtype=torch.bfloat16\n",
    ")\n",
    "\n",
    "model_id = \"garage-bAInd/Platypus2-7B\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=True)\n",
    "model_bf16 = AutoModelForCausalLM.from_pretrained(model_id, device_map=\"auto\", quantization_config=quantization_config)\n",
    "def run_llm(snippets, persons1, persons2):\n",
    "    outputs = []\n",
    "    for snippet, person1, person2 in zip(snippets, persons1, persons2):\n",
    "        pipe = pipeline(\n",
    "        \"text-generation\", model=model_bf16, tokenizer=tokenizer, max_new_tokens=200, temperature = 0.001\n",
    "        )\n",
    "        hf = HuggingFacePipeline(pipeline=pipe)\n",
    "        template = \"\"\"\n",
    "        ####\n",
    "        Instruction: You are a literary scholar.\n",
    "        What is the family relation between {person1} and {person2} in the German text {drama_snippet}?\n",
    "        The possible family relations are parent, child, uncle, siblings, cousins.\n",
    "        Answer in a single sentence in the following format: The family relation between {person1} and {person2} is >>correct family relation<<.\n",
    "        ####\n",
    "        \"\"\"\n",
    "        drama_snippet = str(snippet)\n",
    "        person1 = str(person1)\n",
    "        person2 = str(person2)\n",
    "        prompt = PromptTemplate(\n",
    "            input_variables=[\"drama_snippet\", \"person1\", \"person2\"],\n",
    "            template=template,\n",
    "        )\n",
    "        chain = LLMChain(llm=hf, prompt=prompt)\n",
    "        output = chain.run({\"drama_snippet\": {drama_snippet}, \"person1\": person1, \"person2\": person2})\n",
    "        outputs.append(output)\n",
    "    return outputs\n",
    "df_sample = df.loc[(df.annotation != \"no-transfer\") & (df.relation.isin([\"parent_of\", \"child_of\", \"siblings\", \"uncle_of\", \"cousins\"]))]\n",
    "llm_outputs = run_llm(df_sample.context, df_sample.arg1, df_sample.arg2)\n",
    "predictions_relation = []\n",
    "for o in llm_outputs:\n",
    "    if \"parent\" in o:\n",
    "        prediction_relation = [\"parent_of\"]\n",
    "    elif \"child\" in o:\n",
    "        prediction_relation = [\"child_of\"]\n",
    "    elif \"siblings\" in o:\n",
    "        prediction_relation = [\"siblings\"]\n",
    "    elif \"uncle\" in o:\n",
    "        prediction_relation = [\"uncle_of\"]\n",
    "    elif \"cousins\" in o:\n",
    "        prediction_relation = [\"cousins\"]\n",
    "    predictions_relation.extend(prediction_relation)\n",
    "results_relation = pd.DataFrame(list(zip(list(df_sample.relation), predictions_relation)), columns = [\"key\", \"pred\"])\n",
    "pd.DataFrame(list(zip([sklearn.metrics.f1_score(results_relation.key, results_relation.pred, average=\"weighted\")], \n",
    "             [sklearn.metrics.recall_score(results_relation.key, results_relation.pred, average=\"weighted\")], \n",
    "             [sklearn.metrics.precision_score(results_relation.key, results_relation.pred, average=\"weighted\")],\n",
    "             [sklearn.metrics.accuracy_score(results_relation.key, results_relation.pred)])), \n",
    "             columns = [\"F1\", \"Recall\", \"Precision\", \"Accuracy\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "793669f7-c956-4ac2-88e8-a0e229ad91c8",
   "metadata": {},
   "source": [
    "## 13B - Context window 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e77fb0c5-472a-4f6b-bae7-fbdc70d5bbcd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ad633e3d19db4f258fd6730e3eaa63c7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.8/dist-packages/transformers/generation/configuration_utils.py:362: UserWarning: `do_sample` is set to `False`. However, `temperature` is set to `0.001` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>F1</th>\n",
       "      <th>Recall</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.411558</td>\n",
       "      <td>0.464286</td>\n",
       "      <td>0.49604</td>\n",
       "      <td>0.464286</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         F1    Recall  Precision  Accuracy\n",
       "0  0.411558  0.464286    0.49604  0.464286"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "os.environ['TRANSFORMERS_CACHE'] = \"cache_dir\"\n",
    "import transformers\n",
    "from transformers import pipeline\n",
    "from transformers import BitsAndBytesConfig\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "import re\n",
    "import pandas as pd\n",
    "import torch\n",
    "import langchain\n",
    "from langchain.llms import HuggingFacePipeline\n",
    "from langchain.prompts import PromptTemplate\n",
    "from langchain.chains import LLMChain\n",
    "import sklearn\n",
    "import time\n",
    "from sacrebleu import BLEU\n",
    "import tqdm.contrib\n",
    "df = pd.read_csv(\"./data/KnowledgeTransfers-one_sent.csv\", sep=\",\")\n",
    "quantization_config = BitsAndBytesConfig(\n",
    "    load_in_4bit=True,\n",
    "    bnb_4bit_compute_dtype=torch.bfloat16\n",
    ")\n",
    "\n",
    "model_id = \"garage-bAInd/Platypus2-13B\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=True)\n",
    "model_bf16 = AutoModelForCausalLM.from_pretrained(model_id, device_map=\"auto\", quantization_config=quantization_config)\n",
    "def run_llm(snippets, persons1, persons2):\n",
    "    outputs = []\n",
    "    for snippet, person1, person2 in zip(snippets, persons1, persons2):\n",
    "        pipe = pipeline(\n",
    "        \"text-generation\", model=model_bf16, tokenizer=tokenizer, max_new_tokens=200, temperature = 0.001\n",
    "        )\n",
    "        hf = HuggingFacePipeline(pipeline=pipe)\n",
    "        template = \"\"\"\n",
    "        ####\n",
    "        Instruction: You are a literary scholar.\n",
    "        What is the family relation between {person1} and {person2} in the German text {drama_snippet}?\n",
    "        The possible family relations are parent, child, uncle, siblings, cousins.\n",
    "        Answer in a single sentence in the following format: The family relation between {person1} and {person2} is >>correct family relation<<.\n",
    "        Do NOT write code.\n",
    "        Do NOT write anything before or after the answer sentence.\n",
    "        ####\n",
    "        \"\"\"\n",
    "        drama_snippet = str(snippet)\n",
    "        person1 = str(person1)\n",
    "        person2 = str(person2)\n",
    "        prompt = PromptTemplate(\n",
    "            input_variables=[\"drama_snippet\", \"person1\", \"person2\"],\n",
    "            template=template,\n",
    "        )\n",
    "        chain = LLMChain(llm=hf, prompt=prompt)\n",
    "        output = chain.run({\"drama_snippet\": {drama_snippet}, \"person1\": person1, \"person2\": person2})\n",
    "        outputs.append(output)\n",
    "    return outputs\n",
    "df_sample = df.loc[(df.annotation != \"no-transfer\") & (df.relation.isin([\"parent_of\", \"child_of\", \"siblings\", \"uncle_of\", \"cousins\"]))]\n",
    "llm_outputs = run_llm(df_sample.context, df_sample.arg1, df_sample.arg2)\n",
    "predictions_relation = []\n",
    "for o in llm_outputs:\n",
    "    if \"parent\" in o:\n",
    "        prediction_relation = [\"parent_of\"]\n",
    "    elif \"child\" in o:\n",
    "        prediction_relation = [\"child_of\"]\n",
    "    elif \"siblings\" in o:\n",
    "        prediction_relation = [\"siblings\"]\n",
    "    elif \"uncle\" in o:\n",
    "        prediction_relation = [\"uncle_of\"]\n",
    "    elif \"cousins\" in o:\n",
    "        prediction_relation = [\"cousins\"]\n",
    "    predictions_relation.extend(prediction_relation)\n",
    "results_relation = pd.DataFrame(list(zip(list(df_sample.relation), predictions_relation)), columns = [\"key\", \"pred\"])\n",
    "pd.DataFrame(list(zip([sklearn.metrics.f1_score(results_relation.key, results_relation.pred, average=\"weighted\")], \n",
    "             [sklearn.metrics.recall_score(results_relation.key, results_relation.pred, average=\"weighted\")], \n",
    "             [sklearn.metrics.precision_score(results_relation.key, results_relation.pred, average=\"weighted\")],\n",
    "             [sklearn.metrics.accuracy_score(results_relation.key, results_relation.pred)])), \n",
    "             columns = [\"F1\", \"Recall\", \"Precision\", \"Accuracy\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "204a94da-417c-4ca5-b130-47136ad78ec2",
   "metadata": {},
   "source": [
    "## 13B - Context window 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f837e369-2178-4f23-8ec7-c5a223918bff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c474a4a1151c45fbba772e33ecce7183",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.8/dist-packages/transformers/generation/configuration_utils.py:362: UserWarning: `do_sample` is set to `False`. However, `temperature` is set to `0.001` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>F1</th>\n",
       "      <th>Recall</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.419667</td>\n",
       "      <td>0.464286</td>\n",
       "      <td>0.485267</td>\n",
       "      <td>0.464286</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         F1    Recall  Precision  Accuracy\n",
       "0  0.419667  0.464286   0.485267  0.464286"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "os.environ['TRANSFORMERS_CACHE'] = \"cache_dir\"\n",
    "import transformers\n",
    "from transformers import pipeline\n",
    "from transformers import BitsAndBytesConfig\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "import re\n",
    "import pandas as pd\n",
    "import torch\n",
    "import langchain\n",
    "from langchain.llms import HuggingFacePipeline\n",
    "from langchain.prompts import PromptTemplate\n",
    "from langchain.chains import LLMChain\n",
    "import sklearn\n",
    "import time\n",
    "from sacrebleu import BLEU\n",
    "import tqdm.contrib\n",
    "df = pd.read_csv(\"./data/KnowledgeTransfers-two_sent.csv\", sep=\",\")\n",
    "quantization_config = BitsAndBytesConfig(\n",
    "    load_in_4bit=True,\n",
    "    bnb_4bit_compute_dtype=torch.bfloat16\n",
    ")\n",
    "\n",
    "model_id = \"garage-bAInd/Platypus2-13B\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=True)\n",
    "model_bf16 = AutoModelForCausalLM.from_pretrained(model_id, device_map=\"auto\", quantization_config=quantization_config)\n",
    "def run_llm(snippets, persons1, persons2):\n",
    "    outputs = []\n",
    "    for snippet, person1, person2 in zip(snippets, persons1, persons2):\n",
    "        pipe = pipeline(\n",
    "        \"text-generation\", model=model_bf16, tokenizer=tokenizer, max_new_tokens=200, temperature = 0.001\n",
    "        )\n",
    "        hf = HuggingFacePipeline(pipeline=pipe)\n",
    "        template = \"\"\"\n",
    "        ####\n",
    "        Instruction: You are a literary scholar.\n",
    "        What is the family relation between {person1} and {person2} in the German text {drama_snippet}?\n",
    "        The possible family relations are parent, child, uncle, siblings, cousins.\n",
    "        Answer in a single sentence in the following format: The family relation between {person1} and {person2} is >>correct family relation<<.\n",
    "        Do NOT write code.\n",
    "        Do NOT write anything before or after the answer sentence.\n",
    "        ####\n",
    "        \"\"\"\n",
    "        drama_snippet = str(snippet)\n",
    "        person1 = str(person1)\n",
    "        person2 = str(person2)\n",
    "        prompt = PromptTemplate(\n",
    "            input_variables=[\"drama_snippet\", \"person1\", \"person2\"],\n",
    "            template=template,\n",
    "        )\n",
    "        chain = LLMChain(llm=hf, prompt=prompt)\n",
    "        output = chain.run({\"drama_snippet\": {drama_snippet}, \"person1\": person1, \"person2\": person2})\n",
    "        outputs.append(output)\n",
    "    return outputs\n",
    "df_sample = df.loc[(df.annotation != \"no-transfer\") & (df.relation.isin([\"parent_of\", \"child_of\", \"siblings\", \"uncle_of\", \"cousins\"]))]\n",
    "llm_outputs = run_llm(df_sample.context, df_sample.arg1, df_sample.arg2)\n",
    "predictions_relation = []\n",
    "for o in llm_outputs:\n",
    "    if \"parent\" in o:\n",
    "        prediction_relation = [\"parent_of\"]\n",
    "    elif \"child\" in o:\n",
    "        prediction_relation = [\"child_of\"]\n",
    "    elif \"siblings\" in o:\n",
    "        prediction_relation = [\"siblings\"]\n",
    "    elif \"uncle\" in o:\n",
    "        prediction_relation = [\"uncle_of\"]\n",
    "    elif \"cousins\" in o:\n",
    "        prediction_relation = [\"cousins\"]\n",
    "    predictions_relation.extend(prediction_relation)\n",
    "results_relation = pd.DataFrame(list(zip(list(df_sample.relation), predictions_relation)), columns = [\"key\", \"pred\"])\n",
    "pd.DataFrame(list(zip([sklearn.metrics.f1_score(results_relation.key, results_relation.pred, average=\"weighted\")], \n",
    "             [sklearn.metrics.recall_score(results_relation.key, results_relation.pred, average=\"weighted\")], \n",
    "             [sklearn.metrics.precision_score(results_relation.key, results_relation.pred, average=\"weighted\")],\n",
    "             [sklearn.metrics.accuracy_score(results_relation.key, results_relation.pred)])), \n",
    "             columns = [\"F1\", \"Recall\", \"Precision\", \"Accuracy\"])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}