zero_shot_evaluation_chunker.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# In-context learning for Citation Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dspy\n",
    "import pandas as pd\n",
    "import ast\n",
    "import numpy as np\n",
    "import os\n",
    "from numpy.linalg import norm\n",
    "from tqdm import tqdm\n",
    "from pathlib import Path\n",
    "# from operator import add\n",
    "from PyPDF2 import PdfReader\n",
    "from openai import OpenAI\n",
    "from dspy.evaluate import Evaluate"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get the test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "query_candidate_data = pd.read_csv('darwin/test.qrel.cid', sep=' ', header=None, names=['query', 'candidate', 'bool'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len(query_papers): 115\n",
      "len(candidate_papers): 637\n"
     ]
    }
   ],
   "source": [
    "\n",
    "with open('darwin/qpaper_to_emb', 'r') as f:\n",
    "    query_papers = [line.strip() for line in f]\n",
    "\n",
    "with open('darwin/cpaper_to_emb', 'r') as f:\n",
    "    candidate_papers = [line.strip() for line in f]\n",
    "\n",
    "print(f'len(query_papers): {len(query_papers)}')\n",
    "print(f'len(candidate_papers): {len(candidate_papers)}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     query candidate  bool\n",
      "0  3498240   1824499     1\n",
      "1  3498240  53645322     0\n",
      "2  3498240   1915951     0\n",
      "3  3498240   3048298     0\n",
      "4  3498240   3627503     0\n",
      "Number of query candidate pairs with valid files: 651\n"
     ]
    }
   ],
   "source": [
    "valid_rows = pd.DataFrame()\n",
    "query_dir = 'darwin/query_papers'\n",
    "candidate_dir = 'darwin/candidate_papers'\n",
    "# Iterate over the rows of the data\n",
    "for _, row in query_candidate_data.iterrows():\n",
    "    query_file = os.path.join(query_dir, str(row['query']) + '.pdf')\n",
    "    candidate_file = os.path.join(candidate_dir, str(row['candidate']) + '.pdf')\n",
    "\n",
    "    # Check if both files exist\n",
    "    if os.path.isfile(query_file) and os.path.isfile(candidate_file):\n",
    "        # If both files exist, append the row to valid_rows\n",
    "        valid_rows = valid_rows._append(row)\n",
    "\n",
    "# Reset the index of valid_rows\n",
    "valid_rows.reset_index(drop=True, inplace=True)\n",
    "print(valid_rows.head())\n",
    "print(f'Number of query candidate pairs with valid files: {len(valid_rows)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = [{\"query_file\": query_file, \"candidate_file\": candidate_file, \"cites\": bool(bool_)} for query_file, candidate_file, bool_ in zip(valid_rows['query'], valid_rows['candidate'], valid_rows['bool'])]\n",
    "data = [dspy.Example(**x).with_inputs('query_file', 'candidate_file') for x in data]\n",
    "\n",
    "def split_data(data, split_ratio, seed=42):\n",
    "    np.random.seed(seed)\n",
    "    indices = np.random.permutation(len(data))\n",
    "    split_index = int(split_ratio * len(data))\n",
    "    train_indices = indices[:split_index]\n",
    "    test_indices = indices[split_index:]\n",
    "    trainset = [data[i] for i in train_indices]\n",
    "    testset = [data[i] for i in test_indices]\n",
    "    return trainset, testset\n",
    "\n",
    "# trainset, testset = split_data(data, 0)\n",
    "trainset = data\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Chunker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "llm = dspy.OpenAI(model=\"gpt-3.5-turbo\")\n",
    "dspy.settings.configure(lm=llm, rm=None)\n",
    "\n",
    "client = OpenAI(\n",
    "    # this is also the default, it can be omitted\n",
    "    api_key=os.environ['OPENAI_API_KEY'],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Chunker:\n",
    "    def __init__(self, context_window=3000, max_windows=5):\n",
    "        self.context_window = context_window\n",
    "        self.max_windows = max_windows\n",
    "        self.window_overlap = 0.02\n",
    "\n",
    "    def __call__(self, paper):\n",
    "        snippet_idx = 0\n",
    "\n",
    "        while snippet_idx < self.max_windows and paper:\n",
    "            endpos = int(self.context_window * (1.0 + self.window_overlap))\n",
    "            snippet, paper = paper[:endpos], paper[endpos:]\n",
    "\n",
    "            next_newline_pos = snippet.rfind('\\n')\n",
    "            if paper and next_newline_pos != -1 and next_newline_pos >= self.context_window // 2:\n",
    "                paper = snippet[next_newline_pos+1:] + paper\n",
    "                snippet = snippet[:next_newline_pos]\n",
    "\n",
    "            yield snippet_idx, snippet.strip()\n",
    "            snippet_idx += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DSPy Module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_embeddings(texts, model=\"text-embedding-3-small\", save_file=None):\n",
    "    if save_file and Path(save_file).exists():\n",
    "        with open(save_file, 'r') as f:\n",
    "            # print(f\"Loading embeddings from {save_file}\")\n",
    "            embeddings = [ast.literal_eval(line.strip()) for line in f]\n",
    "        return embeddings\n",
    "        \n",
    "    try:\n",
    "        response = client.embeddings.create(input=texts, model=model)\n",
    "        embeddings = [embedding.embedding for embedding in response.data]\n",
    "        if save_file: # Save the embeddings to a file\n",
    "            with open(save_file, 'w') as f:\n",
    "                # print(f\"Saving embeddings to {save_file}\")\n",
    "                for embedding in embeddings:\n",
    "                    f.write(str(embedding) + '\\n')\n",
    "        return embeddings\n",
    "    except Exception as e:\n",
    "        print(\"Error during API call:\", e)\n",
    "        return []\n",
    "    \n",
    "def get_most_similar_chunk(query_embedding, candidate_embeddings, candidate_chunks):\n",
    "    similarities = np.dot(candidate_embeddings, query_embedding) / (norm(candidate_embeddings, axis=1) * norm(query_embedding))\n",
    "    most_similar_idx = np.argmax(similarities)\n",
    "    return candidate_chunks[most_similar_idx]\n",
    "    \n",
    "    \n",
    "class PredictCitation(dspy.Signature):\n",
    "    __doc__ = \"\"\"Predict if the two chunks are related by a citation. Consider all possible ways in which a citation could occur, such as direct quotes, paraphrasing, or referring to the same ideas or data. Don't be afraid to predict that the chunks are related by a citation. If you're not sure, it's better to predict that they are related.\"\"\"   \n",
    "    query_chunk: str = dspy.InputField(desc='Query chunk to compare to the candidate chunk.')\n",
    "    candidate_chunk: str = dspy.InputField(desc='Candidate chunk to compare to the query chunk.')\n",
    "    answer: bool = dspy.OutputField(desc=\"either True or False\", prefix=\"Answer:\")\n",
    "\n",
    "\n",
    "class PredictCitationAndResolve(dspy.Module):\n",
    "    def __init__(self, context_window=3000, max_windows=5, resolve_function=any,\n",
    "                 candidate_folder='darwin/candidate_papers', query_folder='darwin/query_papers',\n",
    "                 reset_embedding=False):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.chunk = Chunker(context_window=context_window, max_windows=max_windows)\n",
    "        # self.predict = dspy.TypedPredictor(PredictCitation)\n",
    "        # self.predict = dspy.TypedChainOfThought(PredictCitation)\n",
    "        self.predict = dspy.ChainOfThought(PredictCitation)\n",
    "        self.resolve_function = resolve_function\n",
    "        self.query_folder = query_folder\n",
    "        self.candidate_folder = candidate_folder\n",
    "        os.makedirs('embeddings', exist_ok=True)\n",
    "        if reset_embedding:\n",
    "            for emb_file in os.listdir('embeddings'):\n",
    "                os.remove(f'embeddings/{emb_file}')\n",
    "\n",
    "    def forward(self, query_file, candidate_file):\n",
    "        predictions = []\n",
    "        \n",
    "        # Get the text from the pdfs\n",
    "        query_pdf = PdfReader(f'{self.query_folder}/{query_file}.pdf')\n",
    "        query_text = \"\"\n",
    "        for page in query_pdf.pages:\n",
    "            page_text = page.extract_text()\n",
    "            if page_text:\n",
    "                query_text += page_text + \" \"  # Adding space to separate text between pages\n",
    "        query_text = query_text.replace(\"\\n\", \" \")\n",
    "        \n",
    "        candidate_pdf = PdfReader(f'{self.candidate_folder}/{candidate_file}.pdf')\n",
    "        candidate_text = \"\"\n",
    "        for page in candidate_pdf.pages:\n",
    "            page_text = page.extract_text()\n",
    "            if page_text:\n",
    "                candidate_text += page_text + \" \"\n",
    "        candidate_text = candidate_text.replace(\"\\n\", \" \")\n",
    "        \n",
    "        # for each chunk in the paper\n",
    "        query_chunks = [snippet for _, snippet in self.chunk(query_text)]\n",
    "        candidate_chunks = [snippet for _, snippet in self.chunk(candidate_text)]\n",
    "        \n",
    "        # Create embeddings for the chunks\n",
    "        candidate_embeddings = get_embeddings(candidate_chunks, save_file=f'embeddings/candidate_{candidate_file}.emb')\n",
    "        query_embeddings = get_embeddings(query_chunks, save_file=f'embeddings/query_{query_file}.emb')\n",
    "        \n",
    "        for snippet, query_embedding in zip(query_chunks, query_embeddings):\n",
    "            # Get the candidate chunk that is most similar to the snippet\n",
    "            candidate_chunk = get_most_similar_chunk(query_embedding, candidate_embeddings, candidate_chunks)\n",
    "            prediction = self.predict(query_chunk=snippet, candidate_chunk=candidate_chunk)\n",
    "            # print(prediction)\n",
    "            predictions.append(prediction.answer=='True')\n",
    "\n",
    "        return dspy.Prediction(predictions=predictions, resolved=self.resolve_function(predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_chunking = PredictCitationAndResolve(max_windows=15, context_window=1000, reset_embedding=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['JNERJOURNAL OF NEUROENGINEERING AND REHABILITATIONSalazar-Varas et al. Journal of NeuroEngineering and Rehabilitation  (2015) 12:101  DOI 10.1186/s12984-015-0095-4 RESEARCH Open Access Analyzing EEG signals to detect unexpected obstacles during walking R. Salazar-Varas1,Á .C o s t a2,E .I á ñ e z2*,A .Ú b e d a2,E .H o r t a l2a n dJ .M .A z o r í n2 Abstract Background: When an unexpected perturbation in the environment occurs, the subsequent alertness state may cause a brain activation responding to that perturbation which can be detected and employed by a Brain-Computer Interface (BCI). In this work, the possibility of detecting a sudden obstacle appearance analyzing electroencephalographic (EEG) signals is assessed. For this purpose, different features of EEG signals are evaluated during the appearance of sudden obstacles while a subject is walking on a treadmill. The future goal is to use this procedure to detect any obstacle appearance during walking when the user is wearing a lower limb exoskeleton', 'in order to generate an emergency stop command for the exoskeleton. This would enhance the user-exoskeleton interaction, improving the safety mechanisms of current exoskeletons. Methods: In order to detect the change in the brain activity when an obstacle suddenly appears, different features of EEG signals are evaluated using the recordings of five healthy subjects. Since the change in the brain activity occurs in the time domain, the features evaluated are: common spatial patterns, average power, slope, and the coefficients of a polynomial fit. A Linear Discriminant Analysis-based classifier is used to differentiate between two conditions: the appearance or not of an obstacle. The evaluation of the performance to detect the obstacles is made in terms of accuracy, true positive (TP) and false positive (FP) rates. Results: From the offline analysis, the best performance is achieved when the slope or the polynomial coefficients are used as features, with average detection accuracy rates of 74.0 and 79.5 %,', 'respectively. These results are consistent with the pseudo-online results, where a complete EEG recording is segmented into windows of 500 ms and overlapped 400 ms, and a decision about the obstacle appearance is made for each window. The results of the best subject were 11 out of 14 obstacles detected with a rate of 9.09 FPs/min, and 10 out of 14 obstacles detected with a rate of 6.34 FPs/min using slope and polynomial coefficients features, respectively. Conclusions: An EEG-based BCI can be developed to detect the appearance of unexpected obstacles. The average accuracy achieved is 79.5 % of success rate with a low number of false detections. Thus, the online performance of the BCI would be suitable for commanding in a safely way a lower limb exoskeleton during walking. Keywords: BCI, Obstacle detection, Lower limb rehabilitation, EEG, Wearable robot, Exoskeleton Background Brain activity can be classified as evoked or spontaneous depending on the volitional capability of the user to con- trol it. For', 'instance, the performance of real and imaginary movements induces changes in the brain activity that can be controlled by the user, while the perception of *Correspondence: eianez@umh.es 2Brain-Machine Interface Systems Lab, Universidad Miguel Hernández de Elche, Av. de la Universidad, S/N, 03202 Elche, Spain Full list of author information is available at the end of the articlea visual, auditory or sensitive stimulus provokes auto- matic changes in the brain potentials, called event-related potentials (ERP) [1]. Some ERPs are related to an alert- ness state like contingent-negative variation (CNV) [2]. It occurs when a warning stimulus (W1) appears followed by an imperative stimulus (W2) that requires a mental or motor response. When the interval between W1 and W2 is more than two seconds it is possible to distinguish two CNV components: 1) an early component registered over the fronto-central area with bigger amplitude between 550 and 700 ms after the warning stimulus W1, and © 2015 Salazar-Varas et al.', 'Open Access This article is distributed under the terms of the Creative Commons Attribution 4.0 International License (http://creativecommons.org/licenses/by/4.0/), which permits unrestricted use, distribution, and reproduction in any medium, provided you give appropriate credit to the original author(s) and the source, provide a link to the Creative Commons license, and indicate if changes were made. The Creative Commons Public Domain Dedication waiver (http://creativecommons.org/publicdomain/zero/1.0/) applies to the data made available in this article, unless otherwise stated. Salazar-Varas et al. Journal of NeuroEngineering and Rehabilitation  (2015) 12:101 Page 2 of 15 2) the late component which occurs in a centro-parietal location with its maximum amplitude occurring 200 ms before the imperative stimulus W2. The early CNV reflects the processing of the warning signal and the antic- ipation of the upcoming event, the late CNV involves neural activity prior to reaction [3, 4]. These changes in the b', 'rain activity, which can be recorded in a non-invasive way through electroen- cephalography (EEG), can be used to develop a Brain- Computer Interface (BCI) to establish a communication pathway between a subject and a device without the use of peripheral nerves [5, 6]. Until the last decade, BCI applications in the rehabilitation field were limited to the control of upper limb prosthetics and orthotics [7, 8]. More recently, it has been demonstrated that informa- tion about locomotion can be extracted from the pri- mary motor cortex [9], and the first efforts to decode gait parameters from EEG recordings are being devel- oped [10, 11]. In this sense, BCIs have started to be used in the rehabilitation of lower limbs, being used to command exoskeletons. An exoskeleton is an assis- tive device that helps the user in movement execution tasks [12]. Current approaches to command exoskele- tons only consider a planned and voluntary control of the start and stop of the gait based on sensorimotor rhythms [13–16]. A', 'lso, brain activity has been analyzed when there are volitional changes in the gait speed [17]. The results of that work suggest that it is possible to detect a volitional change in gait speed and that the pari- etal cortex may be involved in motor planning. In order to command safely an exoskeleton, it would be desirable that the exoskeleton would implement a procedure to detect the sudden appearance of any obstacle. Although this procedure could be developed using a camera-based system, achieving this detection through EEG signals would imply a greater involvement of the user during the gait process. As a consequence, user-exoskeleton inter- action would improve. Furthermore, if the obstacle were detected through EEG signals, the user wold be able to volitionally discriminate if the obstacle is dangerous or not. The interaction between brain and spinal neuronal activity during the preparation and performance of obsta- cle avoidance has been previously investigated [18]. In that work, subjects were acous', 'tically informed about an approaching obstacle. Subjects avoided the obstacle step- ping over it with the right foot. That means the avoidance of the obstacle was always in the swing of the right leg. However, in real life the obstacle appearance is unex- pected and the subjects must react immediately regardless the phase of the gait in which they are. Moreover, the obstacle cannot always be avoided. The change of the EEG activity when an obstacle sud- denly appears while a subject is walking was assessed inour prior work [19]. Different responses to the obstacle appearance were evaluated: to avoid the obstacle (sub- jects stop their walking), to ignore the obstacle appear- ance (subjects do not stop their walking), to avoid the obstacle with a delay (subjects stop their walking a few seconds after the obstacle appears), and to react without obstacle appearance (subjects stop their walk- ing when they want to). The results obtained suggested that there is a change in the EEG potential over the fronto-cent', 'ral area when the subjects react to avoid the obstacle. T h eg o a lo ft h ep r e s e n tw o r ki st oe v a l u a t ei fi ti sp o s - sible to detect the obstacle appearance from the change in the brain activity. Different features of the EEG sig- nals are evaluated separately in order to select one that allows detecting the appearance of an unexpected obsta- cle during walking previous to subject’s reaction. For each feature, different time intervals are evaluated in order to obtain the optimal in terms of the performance in the dis- crimination task. This means, the optimal time interval is used to extract the feature to train the classifier. Not only is the BCI performance in the detection of obsta- cles evaluated offline, but a pseudo-online analysis is also used to evaluate the performance in similar conditions to real time. The final application will be to send a con- trol command to stop the exoskeleton when an obstacle is detected. This work is part of the BioMot Project (Smart Wearable Robots wit', 'h Bioinspired Sensory-Motor Skills), funded by the Commission of the European Union under Grant Agreement number IFP7-ICT- 2013-10-611695 [20]. The project is focused, among others aspects, on developing a wearable robot (WR) that uses EEG information as part of the control system in order to involve the user in the rehabilitation therapy. Methods Experimental set-up EEG acquisition equipment The EEG signals are acquired using the commercial ampli- fier g.USBamp of the g.Tec company with the active electrodes g.LADYbird to improve signal/noise ratio. The acquisition of EEG signals is done using 32 electrodes placed over the scalp with the following distribution: Fz, FCz, FC5, FC3, FC1, FC6, FC4, FC2, Cz, C5, C3, C1, C6, C4, C2, CPz, CP5, CP3, CP1, CP6, CP4, CP2, Pz, P3, P1, P4 , P2, POz, PO7, PO3, PO8 and PO4, according to the International 10/10 System. Signals are digitalized at a sampling frequency of 1200 Hz. Due to technical complications, some registers were performed with the ActiCHamp equipment of', 'the BrainProducts company. The same distribution of elec- trodes was used and the signals were registered with a sampling frequency of 500 Hz. Salazar-Varas et al. Journal of NeuroEngineering and Rehabilitation  (2015) 12:101 Page 3 of 15 Inertial measurement units During the tests, kinematic information is also recorded in order to know when the subject has reacted to the obsta- cle appearance. Seven inertial measurement units (IMUs) are located on the lower limbs, three on each limb (ankle, calf muscle and quadriceps), and one on the lumbar region. Each IMU registers 19 parameters: rotation matrix (nine parameters), acceleration matrix (three parameters, m/s2), angular velocity matrix (three parameters, rad/s), magnetic field (three parameters) and temperature. The sampling frequency is 30 Hz. Additional equipment To achieve a constant velocity during the gait, a tread- mill Pro-form Performance 750 is used. Two procedures are used to simulate the appearance of an obstacle during walking on the treadmi', 'll. In the first one, a line laser is pro- jected over the treadmill to simulate the appearance of the obstacle. The laser module emits a wavelength of 635 nm (red color), with an output power of 3 mW. In the second one, a screen placed in front of the treadmill changes its color to simulate the appearance of the obstacle. Experimental procedure To use the information recorded by IMUs in the EEG analysis, a synchronized register of EEG and kinematics is necessary. So, a software based on Matlab has been devel- oped allowing the synchronized register. The software allows connecting both systems by using the Application Programming Interface (API) provided. First, a configura- tion of both is performed. Then the experiment starts by asking the subject to stand still on the treadmill for a few seconds while the IMUs are calibrated. Once this calibra- tion finishes, the subject starts walking on the treadmill with a constant velocity of 2 km/h and 0 degrees of incli- nation. As it has been indicated previousl', 'y, the obstacle appearance is simulated in two ways: 1) The projection of a line over the treadmill (using a laser) interfering with the subject’s gait (labeled as Laser mode); 2) The change of the background color of a screen placed in front of the treadmill (labeled as Screen mode). A repre- sentation of the experimental environment is shown in Fig. 1. Different responses regarding the obstacle appearance are requested to the subject: •Reaction . The subject reacts to the obstacle stopping the gait for a moment and then starts the gait again. If the subject stops the gait, the treadmill has enough space to move backwards during a short period of time. •Delayed reaction . The subject continues walking two or three steps after the obstacle appearance, stops the gait for a moment and then starts the gait again. Fig. 1 Experimental environment. The subject is walking on the treadmill while the EEG signals are recorded. For the Laser mode, a line is projected over the treadmill (using a laser) while the subj', 'ect is walking. For the Screen mode, the background color of the screen placed in front of the user changes •No reaction . The subject ignores the obstacle appearance and continues walking normally. •Free reaction . The subject freely decides when to stop the gait several times during the recording. In the case of Reaction andNo reaction , the obstacle rep- resentation (i.e. Laser and Screen) is held during 5 s, while in the case of Delayed reaction is held during 10 s. A run consists of: 180 s of Reaction condition, 240 s of Delayed reaction condition, 180 s of No reaction condi- tion and 120 s of Free reaction condition. In all cases, the obstacle appears seven times for each obstacle representa- tion. In order to avoid subject’s prediction of the obstacle apparition, the inter obstacle time has a random value between 2–5 s. This procedure has been tested in five male healthy subjects (labeled as S1, S2, S3, S4, S5) with ages between 24 and 29 years, all of them right handed, without any neurological di', 'sorder and with normal vision. The subjects have performed a total of four runs. The experiments were approved by the Ethics Committee of the Miguel Hernandez University of Elche (Spain). All Salazar-Varas et al. Journal of NeuroEngineering and Rehabilitation  (2015) 12:101 Page 4 of 15 subjects were informed and signed an informed consent according to Helsinski declaration. The register of S1, S2 and S3 was carried out with the g.Tec, while the register of S4 and S5 was performed with the BrainProducts equipment. For these two subjects only Screen obstacle representation was used. All four different responses were used in our previous work [19] to evaluate how the EEG signals changed in each case. However, in this work only the Reaction data are used since the goal is to detect the obstacle appearance from EEG signals when the subject reacts to the obstacle (actually before the subject stops their gait). EEG signal processing and analysis TheReaction data have been analyzed offline. The chan- nels FC5, F']\n"
     ]
    }
   ],
   "source": [
    "chunker = Chunker(context_window=1000, max_windows=15)\n",
    "query_pdf = PdfReader(f'darwin/query_papers/1323414.pdf')\n",
    "query_text = \"\"\n",
    "for page in query_pdf.pages:\n",
    "    page_text = page.extract_text()\n",
    "    if page_text:\n",
    "        query_text += page_text + \" \"  # Adding space to separate text between pages\n",
    "query_text = query_text.replace(\"\\n\", \" \")\n",
    "query_chunks = [snippet for _, snippet in chunker(query_text)]\n",
    "print(query_chunks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1020\n",
      "15\n"
     ]
    }
   ],
   "source": [
    "print(len(query_chunks[0]))\n",
    "print(len(query_chunks))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Example({'query_file': 1323414, 'candidate_file': '3324808'}) (input_keys=None)\n",
      "Example({'cites': False}) (input_keys=None)\n",
      "Prediction(\n",
      "    predictions=[False, False, False, True, False, False, False, False, True, False, True, True, False, False, False],\n",
      "    resolved=True\n",
      ")\n",
      "False\n"
     ]
    }
   ],
   "source": [
    "# get an example\n",
    "example = trainset[-2]\n",
    "example_x = example.inputs()\n",
    "example_y = example.labels()\n",
    "print(example_x)\n",
    "print(example_y)\n",
    "\n",
    "prediction = pipeline_chunking(**example_x)\n",
    "print(prediction)\n",
    "print(example_y.cites)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "Predict if the two chunks are related by a citation. Consider all possible ways in which a citation could occur, such as direct quotes, paraphrasing, or referring to the same ideas or data. Don't be afraid to predict that the chunks are related by a citation. If you're not sure, it's better to predict that they are related.\n",
      "\n",
      "---\n",
      "\n",
      "Follow the following format.\n",
      "\n",
      "Query Chunk: Query chunk to compare to the candidate chunk.\n",
      "\n",
      "Candidate Chunk: Candidate chunk to compare to the query chunk.\n",
      "\n",
      "Reasoning: Let's think step by step in order to ${produce the answer}. We ...\n",
      "\n",
      "Answer: either True or False\n",
      "\n",
      "---\n",
      "\n",
      "Query Chunk: the BrainProducts company. The same distribution of elec- trodes was used and the signals were registered with a sampling frequency of 500 Hz. Salazar-Varas et al. Journal of NeuroEngineering and Rehabilitation (2015) 12:101 Page 3 of 15 Inertial measurement units During the tests, kinematic information is also recorded in order to know when the subject has reacted to the obsta- cle appearance. Seven inertial measurement units (IMUs) are located on the lower limbs, three on each limb (ankle, calf muscle and quadriceps), and one on the lumbar region. Each IMU registers 19 parameters: rotation matrix (nine parameters), acceleration matrix (three parameters, m/s2), angular velocity matrix (three parameters, rad/s), magnetic field (three parameters) and temperature. The sampling frequency is 30 Hz. Additional equipment To achieve a constant velocity during the gait, a tread- mill Pro-form Performance 750 is used. Two procedures are used to simulate the appearance of an obstacle during walking on the treadmi\n",
      "\n",
      "Candidate Chunk: 013a environment. For each subject, echo-planar images subsequently underwent steps for slice-timing correction, spatial realign-ment, and registration to standardized MNI space carried out with a population template generated from the T1 structural images using the Diffeomorphic Anatomical Registration Through Exponentiated Lie Algebra algorithm [Ashburner, 2007]. Spurious variance was reduced by regressing-out signal from white matter, cerebrospinal ﬂuid, and by voxel-speciﬁc motion correction [Sat- terthwaite et al., 2013]. Next, the images were detrended and band-pass ﬁltered (0.01–0.1 Hz) to eliminate biologi-cally non-relevant signals and the resulting output was used to calculate sample entropy [Liu et al., 2013; Richman and Moorman, 2000] at the voxel-level, as implemented in the “complexity” toolbox (LOFT Lab). Sample entropy is formally deﬁned as the negative loga- rithm of the conditional probability that if two sets of simultaneous data points (vector pairs) with length m meet similarity crite\n",
      "\n",
      "Reasoning: Let's think step by step in order to\u001b[32m produce the answer. We can see that both chunks discuss the equipment and procedures used in a study. The first chunk talks about the use of inertial measurement units and a treadmill during tests, while the second chunk discusses the steps taken with echo-planar images for data analysis. Both chunks provide specific details about the equipment and procedures used in their respective studies. Therefore, it is possible that the second chunk is related to the first chunk by citation, as it could be referencing the methods used in the study described in the first chunk.\n",
      "\n",
      "Answer: True\u001b[0m\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Predict if the two chunks are related by a citation. Consider all possible ways in which a citation could occur, such as direct quotes, paraphrasing, or referring to the same ideas or data. Don't be afraid to predict that the chunks are related by a citation. If you're not sure, it's better to predict that they are related.\n",
      "\n",
      "---\n",
      "\n",
      "Follow the following format.\n",
      "\n",
      "Query Chunk: Query chunk to compare to the candidate chunk.\n",
      "\n",
      "Candidate Chunk: Candidate chunk to compare to the query chunk.\n",
      "\n",
      "Reasoning: Let's think step by step in order to ${produce the answer}. We ...\n",
      "\n",
      "Answer: either True or False\n",
      "\n",
      "---\n",
      "\n",
      "Query Chunk: ll. In the first one, a line laser is pro- jected over the treadmill to simulate the appearance of the obstacle. The laser module emits a wavelength of 635 nm (red color), with an output power of 3 mW. In the second one, a screen placed in front of the treadmill changes its color to simulate the appearance of the obstacle. Experimental procedure To use the information recorded by IMUs in the EEG analysis, a synchronized register of EEG and kinematics is necessary. So, a software based on Matlab has been devel- oped allowing the synchronized register. The software allows connecting both systems by using the Application Programming Interface (API) provided. First, a configura- tion of both is performed. Then the experiment starts by asking the subject to stand still on the treadmill for a few seconds while the IMUs are calibrated. Once this calibra- tion finishes, the subject starts walking on the treadmill with a constant velocity of 2 km/h and 0 degrees of incli- nation. As it has been indicated previousl\n",
      "\n",
      "Candidate Chunk: 013a environment. For each subject, echo-planar images subsequently underwent steps for slice-timing correction, spatial realign-ment, and registration to standardized MNI space carried out with a population template generated from the T1 structural images using the Diffeomorphic Anatomical Registration Through Exponentiated Lie Algebra algorithm [Ashburner, 2007]. Spurious variance was reduced by regressing-out signal from white matter, cerebrospinal ﬂuid, and by voxel-speciﬁc motion correction [Sat- terthwaite et al., 2013]. Next, the images were detrended and band-pass ﬁltered (0.01–0.1 Hz) to eliminate biologi-cally non-relevant signals and the resulting output was used to calculate sample entropy [Liu et al., 2013; Richman and Moorman, 2000] at the voxel-level, as implemented in the “complexity” toolbox (LOFT Lab). Sample entropy is formally deﬁned as the negative loga- rithm of the conditional probability that if two sets of simultaneous data points (vector pairs) with length m meet similarity crite\n",
      "\n",
      "Reasoning: Let's think step by step in order to\u001b[32m produce the answer. We can see that both chunks describe experimental procedures and data processing steps in scientific research. The first chunk discusses the use of a line laser and software for synchronized EEG and kinematics analysis, while the second chunk talks about image processing steps for MRI data analysis. While they do not directly cite each other, they both provide detailed information on experimental procedures and data processing methods in scientific studies.\n",
      "\n",
      "Answer: True\u001b[0m\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Predict if the two chunks are related by a citation. Consider all possible ways in which a citation could occur, such as direct quotes, paraphrasing, or referring to the same ideas or data. Don't be afraid to predict that the chunks are related by a citation. If you're not sure, it's better to predict that they are related.\n",
      "\n",
      "---\n",
      "\n",
      "Follow the following format.\n",
      "\n",
      "Query Chunk: Query chunk to compare to the candidate chunk.\n",
      "\n",
      "Candidate Chunk: Candidate chunk to compare to the query chunk.\n",
      "\n",
      "Reasoning: Let's think step by step in order to ${produce the answer}. We ...\n",
      "\n",
      "Answer: either True or False\n",
      "\n",
      "---\n",
      "\n",
      "Query Chunk: y, the obstacle appearance is simulated in two ways: 1) The projection of a line over the treadmill (using a laser) interfering with the subject’s gait (labeled as Laser mode); 2) The change of the background color of a screen placed in front of the treadmill (labeled as Screen mode). A repre- sentation of the experimental environment is shown in Fig. 1. Different responses regarding the obstacle appearance are requested to the subject: •Reaction . The subject reacts to the obstacle stopping the gait for a moment and then starts the gait again. If the subject stops the gait, the treadmill has enough space to move backwards during a short period of time. •Delayed reaction . The subject continues walking two or three steps after the obstacle appearance, stops the gait for a moment and then starts the gait again. Fig. 1 Experimental environment. The subject is walking on the treadmill while the EEG signals are recorded. For the Laser mode, a line is projected over the treadmill (using a laser) while the subj\n",
      "\n",
      "Candidate Chunk: s signiﬁcantanxiety. His behavioral data, however, was collected andthe pattern of personality change was consistent with whatwas observed at the group level. Thus, the personality-related effects reported below include this subject. Study Design and Procedures Two scanning sessions with either 75 mg of LSD or placebo (each given intravenously over two minutes) were scheduledfor each participant, in a counter-balanced order, with aninterval of at least two weeks between each session (Support- ing Information Figure S1). For each session, three 7.5-min BOLD resting-state fMRI scans were acquired (lasting 25 min,in total): resting state 1 (no music), resting state 2 (music), andresting state 3 (no music). For the ﬁrst and third states, thesubjects were instructed to rest quietly and keep their eyesclosed. Instructions were the same for the second state, exceptthat the subjects were informed that they would listen toambient music played through headphones. Two tracks fromthe album “Yearning” by Robert Rich a\n",
      "\n",
      "Reasoning: Let's think step by step in order to\u001b[32m produce the answer. We can see that the query chunk discusses the experimental environment and the different responses requested from the subject when faced with an obstacle. On the other hand, the candidate chunk talks about the study design and procedures for scanning sessions with LSD or placebo. There is no direct connection between the content of the two chunks, so they are not related by a citation.\n",
      "\n",
      "Answer: False\u001b[0m\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Predict if the two chunks are related by a citation. Consider all possible ways in which a citation could occur, such as direct quotes, paraphrasing, or referring to the same ideas or data. Don't be afraid to predict that the chunks are related by a citation. If you're not sure, it's better to predict that they are related.\n",
      "\n",
      "---\n",
      "\n",
      "Follow the following format.\n",
      "\n",
      "Query Chunk: Query chunk to compare to the candidate chunk.\n",
      "\n",
      "Candidate Chunk: Candidate chunk to compare to the query chunk.\n",
      "\n",
      "Reasoning: Let's think step by step in order to ${produce the answer}. We ...\n",
      "\n",
      "Answer: either True or False\n",
      "\n",
      "---\n",
      "\n",
      "Query Chunk: ect is walking. For the Screen mode, the background color of the screen placed in front of the user changes •No reaction . The subject ignores the obstacle appearance and continues walking normally. •Free reaction . The subject freely decides when to stop the gait several times during the recording. In the case of Reaction andNo reaction , the obstacle rep- resentation (i.e. Laser and Screen) is held during 5 s, while in the case of Delayed reaction is held during 10 s. A run consists of: 180 s of Reaction condition, 240 s of Delayed reaction condition, 180 s of No reaction condi- tion and 120 s of Free reaction condition. In all cases, the obstacle appears seven times for each obstacle representa- tion. In order to avoid subject’s prediction of the obstacle apparition, the inter obstacle time has a random value between 2–5 s. This procedure has been tested in five male healthy subjects (labeled as S1, S2, S3, S4, S5) with ages between 24 and 29 years, all of them right handed, without any neurological di\n",
      "\n",
      "Candidate Chunk: s signiﬁcantanxiety. His behavioral data, however, was collected andthe pattern of personality change was consistent with whatwas observed at the group level. Thus, the personality-related effects reported below include this subject. Study Design and Procedures Two scanning sessions with either 75 mg of LSD or placebo (each given intravenously over two minutes) were scheduledfor each participant, in a counter-balanced order, with aninterval of at least two weeks between each session (Support- ing Information Figure S1). For each session, three 7.5-min BOLD resting-state fMRI scans were acquired (lasting 25 min,in total): resting state 1 (no music), resting state 2 (music), andresting state 3 (no music). For the ﬁrst and third states, thesubjects were instructed to rest quietly and keep their eyesclosed. Instructions were the same for the second state, exceptthat the subjects were informed that they would listen toambient music played through headphones. Two tracks fromthe album “Yearning” by Robert Rich a\n",
      "\n",
      "Reasoning: Let's think step by step in order to\u001b[32m produce the answer. We can see that the first chunk discusses a specific experimental procedure involving obstacle representation and the behavior of subjects during different conditions. On the other hand, the candidate chunk talks about the study design and procedures for scanning sessions with LSD or placebo. There is no direct overlap in content or ideas between the two chunks, so they are not related by a citation.\n",
      "\n",
      "Answer: False\u001b[0m\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Predict if the two chunks are related by a citation. Consider all possible ways in which a citation could occur, such as direct quotes, paraphrasing, or referring to the same ideas or data. Don't be afraid to predict that the chunks are related by a citation. If you're not sure, it's better to predict that they are related.\n",
      "\n",
      "---\n",
      "\n",
      "Follow the following format.\n",
      "\n",
      "Query Chunk: Query chunk to compare to the candidate chunk.\n",
      "\n",
      "Candidate Chunk: Candidate chunk to compare to the query chunk.\n",
      "\n",
      "Reasoning: Let's think step by step in order to ${produce the answer}. We ...\n",
      "\n",
      "Answer: either True or False\n",
      "\n",
      "---\n",
      "\n",
      "Query Chunk: sorder and with normal vision. The subjects have performed a total of four runs. The experiments were approved by the Ethics Committee of the Miguel Hernandez University of Elche (Spain). All Salazar-Varas et al. Journal of NeuroEngineering and Rehabilitation (2015) 12:101 Page 4 of 15 subjects were informed and signed an informed consent according to Helsinski declaration. The register of S1, S2 and S3 was carried out with the g.Tec, while the register of S4 and S5 was performed with the BrainProducts equipment. For these two subjects only Screen obstacle representation was used. All four different responses were used in our previous work [19] to evaluate how the EEG signals changed in each case. However, in this work only the Reaction data are used since the goal is to detect the obstacle appearance from EEG signals when the subject reacts to the obstacle (actually before the subject stops their gait). EEG signal processing and analysis TheReaction data have been analyzed offline. The chan- nels FC5, F\n",
      "\n",
      "Candidate Chunk: oraltered states of consciousness questionnaire [Studeruset al., 2010] related to mystical quality of the experiences (unity, spiritual experience, blissful state, and changed meaning of percepts) administered after the scanning ses- sion had been completed (Supporting Information Note 2). Ethics This study was approved by the National Research Ethics Service Committee London–West London and was con- ducted in accordance with the revised declaration of Hel- sinki [2000], the International Committee on Harmonization Good Clinical Practice guidelines and National Health Serv-ice Research Governance Framework. Imperial College Lon- don sponsored the research, which was conducted under a Home Ofﬁce license for research with schedule 1 drugs. Image Preprocessing A standardized pipeline based on “SPM12” (Wellcome Trust Center for Neuroimaging, UCL) was used using the Data Processing Assistant for Resting-State fMRI: Advanced Edition (DPARSFA, version 3.2) [Chao-Gan and Yu-Feng, 2010], installed in the MATLAB R2\n",
      "\n",
      "Reasoning: Let's think step by step in order to\u001b[32m produce the answer. We can see that both chunks mention the approval by an Ethics Committee and the adherence to the Helsinki declaration. Both chunks also mention the use of specific equipment for data collection. However, the first chunk specifically mentions the use of EEG signals to detect obstacle appearance, while the second chunk discusses the administration of a questionnaire related to altered states of consciousness after a scanning session. These two chunks do not seem to be related by a citation as they discuss different methodologies and data collection processes.\n",
      "\n",
      "Answer: False\u001b[0m\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "llm.inspect_history(n=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "def metric(example, result):\n",
    "    '''Match metric'''\n",
    "    return 1 if example.cites == result.resolved else 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t negative seek value -1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/PyPDF2/_cmap.py:142: PdfReadWarning: Advanced encoding /UniJIS-UCS2-H not implemented yet\n",
      "  warnings.warn(\n",
      "\n",
      "Average Metric: 50.0 / 102  (49.0):  16%|█▌        | 102/651 [06:41<22:25,  2.45s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error during API call: Error code: 400 - {'error': {'message': \"'$.input' is invalid. Please check the API reference: https://platform.openai.com/docs/api-reference.\", 'type': 'invalid_request_error', 'param': None, 'code': None}}\n",
      "Error for example in dev set: \t\t shapes (0,) and (1536,) not aligned: 0 (dim 0) != 1536 (dim 0)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[A\n",
      "Average Metric: 52.0 / 113  (46.0):  17%|█▋        | 113/651 [07:25<34:30,  3.85s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.52s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.67s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t '[' was never closed (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 110.0 / 243  (45.3):  37%|███▋      | 242/651 [15:55<23:41,  3.48s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t '[' was never closed (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.54s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.55s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "\n",
      "Average Metric: 136.0 / 313  (43.5):  48%|████▊     | 313/651 [20:26<17:48,  3.16s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t '[' was never closed (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t '[' was never closed (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 164.0 / 393  (41.7):  60%|██████    | 392/651 [25:24<12:41,  2.94s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t '[' was never closed (<unknown>, line 1)\n",
      "Error for example in dev set: \t\t '[' was never closed (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Average Metric: 164.0 / 395  (41.5):  61%|██████    | 395/651 [25:27<06:57,  1.63s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t '[' was never closed (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[A\n",
      "Average Metric: 164.0 / 397  (41.3):  61%|██████    | 396/651 [25:27<05:56,  1.40s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t '[' was never closed (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t '[' was never closed (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error during API call: Error code: 400 - {'error': {'message': \"'$.input' is invalid. Please check the API reference: https://platform.openai.com/docs/api-reference.\", 'type': 'invalid_request_error', 'param': None, 'code': None}}\n",
      "Error for example in dev set: \t\t shapes (0,) and (1536,) not aligned: 0 (dim 0) != 1536 (dim 0)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.32s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t unmatched ']' (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t unmatched ']' (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t unmatched ']' (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/PyPDF2/_cmap.py:142: PdfReadWarning: Advanced encoding /SymbolSetEncoding not implemented yet\n",
      "  warnings.warn(\n",
      "\n",
      "Average Metric: 216.0 / 519  (41.6):  80%|███████▉  | 519/651 [32:19<07:45,  3.53s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t '[' was never closed (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[A/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.53s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.66s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.50s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.80s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t PyCryptodome is required for AES algorithm\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t negative seek value -1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t leading zeros in decimal integer literals are not permitted; use an 0o prefix for octal integers (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t unmatched ']' (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t unmatched ']' (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t unmatched ']' (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t '[' was never closed (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 270.0 / 651  (41.5): 100%|██████████| 651/651 [40:21<00:00,  3.72s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average Metric: 270.0 / 651  (41.5%)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate = Evaluate(devset=trainset, metric=metric, num_threads=8, display_progress=True, display_table=0, max_errors=100, return_outputs=True)\n",
    "outputs = evaluate(pipeline_chunking)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "651\n"
     ]
    }
   ],
   "source": [
    "all_predictions = []\n",
    "for x in outputs[1]:\n",
    "    if type(x[1])==dspy.Prediction:\n",
    "        all_predictions.append(x[1].resolved)\n",
    "    else:\n",
    "        all_predictions.append(np.nan)\n",
    "    \n",
    "\n",
    "all_labels = [x[0].cites for x in outputs[1]]\n",
    "print(len(all_predictions))\n",
    "\n",
    "with open('darwin/eval/predictions_COT_large_prompt_1000.txt', 'w') as f:\n",
    "    for pred in all_predictions:\n",
    "        f.write(str(pred) + '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.41\n",
      "Recall:  0.70\n",
      "Precision: 0.18\n",
      "F1 Score: 0.29\n"
     ]
    }
   ],
   "source": [
    "# Compute the accuracy of the final predictions\n",
    "correct_predictions = [prediction == label for prediction, label in zip(all_predictions, all_labels)]\n",
    "accuracy = sum(correct_predictions) / len(correct_predictions)\n",
    "print(f'Accuracy: {accuracy:.2f}')\n",
    "\n",
    "# Compute the recall of the final predictions\n",
    "true_positives = sum([prediction and label for prediction, label in zip(all_predictions, all_labels)])\n",
    "false_negatives = sum([not prediction and label for prediction, label in zip(all_predictions, all_labels)])\n",
    "recall = true_positives / (true_positives + false_negatives)\n",
    "print(f'Recall: {recall: .2f}')\n",
    "\n",
    "# Compute the precision of the final predictions\n",
    "true_positives = sum([prediction and label for prediction, label in zip(all_predictions, all_labels)])\n",
    "false_positives = sum([prediction and not label for prediction, label in zip(all_predictions, all_labels)])\n",
    "precision = true_positives / (true_positives + false_positives)\n",
    "print(f'Precision: {precision:.2f}')\n",
    "\n",
    "# F1 score\n",
    "f1 = 2 * (precision * recall) / (precision + recall)\n",
    "print(f'F1 Score: {f1:.2f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " True,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " True,\n",
       " False,\n",
       " False,\n",
       " nan,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " nan,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " True,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " True,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " nan,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " nan,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " nan,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " False,\n",
       " False,\n",
       " nan,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " nan,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " nan,\n",
       " False,\n",
       " nan,\n",
       " False,\n",
       " nan,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " True,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " True,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " nan,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " nan,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " True,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " nan,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " nan,\n",
       " False,\n",
       " False,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " nan,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False,\n",
       " False]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Weird paper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'237TomasMikolov,QuocVLe,andIlyaSutskever.2013b.\\nExploitingsimilaritiesamonglanguagesformachine\\ntranslation. CoRR.\\nTomas Mikolov, Ilya Sutskever, Kai Chen, Gregory S.\\nCorrado, and Jeffrey Dean. 2013c. Distributed rep-\\nresentations of words and phrases and their compo-\\nsitionality. In NIPS.\\nDavid Milne and Ian H. Witten. 2008. An effective,\\nlow-cost measure of semantic relatedness obtained\\nfromwikipedialinks. In AAAI.\\nMike Mintz, Steven Bills, Rion Snow, and Daniel Ju-\\nrafsky.2009. Distantsupervisionforrelationextrac-\\ntionwithoutlabeleddata. In ACL/IJCNLP .\\nAditya Mogadala and Achim Rettinger. 2016. Bilin-\\ngual word embeddings from parallel and non-\\nparallel corpora for cross-language text classiﬁca-\\ntion. In HLT-NAACL .\\nThien Huu Nguyen, Nicolas Fauceglia, Mariano Ro-\\ndriguez Muro, Oktie Hassanzadeh, Alﬁo Massimil-\\nianoGliozzo,andMohammadSadoghi.2016. Joint\\nlearning of local and global features for entity link-\\ningvianeuralnetworks. In COLING.\\nSebastian Ruder, Ivan Vulić, and Anders Søgaard.\\n2017. A survey of cross-lingual word embedding\\nmodels. arXiv preprint arXiv:1706.04902 .\\nTianzeShi,ZhiyuanLiu,YangLiu,andMaosongSun.\\n2015. Learning cross-lingual word embeddings via\\nmatrixco-factorization. In ACL.\\nRadu Soricut and Nan Ding. 2016. Multilingual word\\nembeddingsusingmultigraphs. CoRR.\\nMihai Surdeanu, Julie Tibshirani, Ramesh Nallapati,\\nand Christopher D. Manning. 2012. Multi-instance\\nmulti-label learning for relation extraction. In\\nEMNLP-CoNLL .\\nKristina Toutanova, Danqi Chen, Patrick Pantel, Hoi-\\nfungPoon,PallaviChoudhury,andMichaelGamon.\\n2015. Representingtextforjointembeddingoftext\\nandknowledgebases. In EMNLP.\\nIvan Vulic and Marie-Francine Moens. 2015. Bilin-\\ngualwordembeddingsfromnon-paralleldocument-\\naligned data applied to bilingual lexicon induction.\\nInACL.\\nIvanVulicandMarie-FrancineMoens.2016. Bilingual\\ndistributed word representations from document-\\nalignedcomparabledata. JAIR.\\nZhen Wang, Jianwen Zhang, Jianlin Feng, and Zheng\\nChen. 2014. Knowledge graph and text jointly em-\\nbedding. In EMNLP.\\nZhigang Wang and Juan-Zi Li. 2016. Text-enhanced\\nrepresentationlearningforknowledgegraph. In IJ-\\nCAI.Jason Weston, Antoine Bordes, Oksana Yakhnenko,\\nand Nicolas Usunier. 2013a. Connecting language\\nandknowledgebaseswithembeddingmodelsforre-\\nlationextraction. In ACL.\\nJason Weston, Antoine Bordes, Oksana Yakhnenko,\\nand Nicolas Usunier. 2013b. Connecting language\\nandknowledgebaseswithembeddingmodelsforre-\\nlationextraction. In EMNLP.\\nHaiyang Wu, Daxiang Dong, Xiaoguang Hu, Dian-\\nhai Yu, Wei He, Hua Wu, Haifeng Wang, and Ting\\nLiu. 2014. Improve statistical machine translation\\nwithcontext-sensitivebilingualsemanticembedding\\nmodel. In EMNLP.\\nJiawei Wu, Ruobing Xie, Zhiyuan Liu, and Maosong\\nSun. 2016. Knowledge representation via joint\\nlearning of sequential text and knowledge graphs.\\nCoRR.\\nMin Xiao and Yuhong Guo. 2014. Distributed word\\nrepresentationlearningforcross-lingualdependency\\nparsing. In CoNLL.\\nIkuya Yamada, Hiroyuki Shindo, Hideaki Takeda, and\\nYoshiyasuTakefuji.2016. Jointlearningoftheem-\\nbedding of words and entities for named entity dis-\\nambiguation. In CoNLL.\\nIkuya Yamada, Hiroyuki Shindo, Hideaki Takeda, and\\nYoshiyasuTakefuji.2017. Learningdistributedrep-\\nresentations of texts and entities from knowledge\\nbase. TACL.\\nBishan Yang and Tom M. Mitchell. 2017. Leverag-\\ningknowledgebasesinlstmsforimprovingmachine\\nreading. In ACL.\\nDaojian Zeng, Kang Liu, Yubo Chen, and Jun Zhao.\\n2015. Distant supervision for relation extraction\\nvia piecewise convolutional neural networks. In\\nEMNLP.\\nJingZhang,YixinCao,LeiHou,JuanziLi,andHai-Tao\\nZheng.2017a. Xlink: anunsupervisedbilingualen-\\ntity linking system. In Chinese Computational Lin-\\nguistics and Natural Language Processing Based on\\nNaturally Annotated Big Data .\\nMeng Zhang, Yang Liu, Huanbo Luan, and Maosong\\nSun. 2017b. Adversarial training for unsupervised\\nbilinguallexiconinduction. In ACL.\\nYuan Zhang, David Gaddy, Regina Barzilay, and\\nTommi S. Jaakkola. 2016. Ten pairs to tag - multi-\\nlingualpostaggingviacoarsemappingbetweenem-\\nbeddings. In HLT-NAACL .\\nHao Zhu, Ruobing Xie, Zhiyuan Liu, and Maosong\\nSun. 2017. Iterative entity alignment via joint\\nknowledgeembeddings. In IJCAI.\\nWill Y. Zou, Richard Socher, Daniel M. Cer, and\\nChristopherD.Manning.2013. Bilingualwordem-\\nbeddings for phrase-based machine translation. In\\nEMNLP.'"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "PdfReader('darwin/query_papers/53079158.pdf').pages[-1].extract_text()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "Predict if the two chunks are related by a citation. Consider all possible ways in which a citation could occur, such as direct quotes, paraphrasing, or referring to the same ideas or data. Don't be afraid to predict that the chunks are related by a citation. If you're not sure, it's better to predict that they are related.\n",
      "\n",
      "---\n",
      "\n",
      "Follow the following format.\n",
      "\n",
      "Query Chunk: Query chunk to compare to the candidate chunk.\n",
      "Candidate Chunk: Candidate chunk to compare to the query chunk.\n",
      "Answer: either True or False\n",
      "\n",
      "---\n",
      "\n",
      "Query Chunk: s/article/19/16/2088/242445 by guest on 05 April 2024 S.Oba et al. Factorscores x=(x1,...,xK)fortheexpressionvector y are obtained by minimization of the residual error: err=/vextenddouble/vextenddouble/vextenddoubleyobs−Wobsx/vextenddouble/vextenddouble/vextenddouble2 . Thisisawell-knownregressionproblem,andtheleastsquare solution is given by x=(WobsTWobs)−1WobsTyobs. Using x, the missing part is estimated as ymiss=Wmissx.( 2 ) InthePCregressionabove, Wshouldbeknownbeforehand. Later, we will discuss the way to determine the parameter. 2.3 Bayesian estimation A parametric probabilistic model, which is called probab-ilistic PCA (PPCA), has been proposed recently (Tippingand Bishop, 1999). The probabilistic model is based on theassumption that the residual error /epsilon1and the factor scores x l(1≤l≤K)in Equation (1) obey normal distributions: p(x)=NK(x|0,IK), p(/epsilon1)=ND(/epsilon1|0,(1/τ)ID), where NK(x|µ,/Sigma1)denotes a K-dimensional normal distri- bution for x, whose mean and covariance are µand/\n",
      "Candidate Chunk: cular diseases such as atherosclerosis, obesity, type 2 diabetes and cardiac remodeling. Thus, the effects ofIL-33 are either pro- or anti-inflammatory depending on the disease and the model. In this review the role of IL-33 in the inflammation of several disease pathologies will be discussed, with particular emphasis on recent advances. Review Basic Biology of IL-33 Interleukin (IL)-33 (also known as IL-1F11) was origin- ally identified as DVS27, a gene up-regulated in caninecerebral vasospasm [1], and as “nuclear factor from high endothelial venules ”(NF-HEV) [2]. However, in 2005 analysis of computational structural databases revealedthat this protein had close amino acid homology toIL-18, and a b-sheet trefoil fold structure characteristic of IL-1 family members [3]. IL-33 binds to a ST2L (alsoknown as T1, IL-1RL1, DER4), which is a member ofthe Toll-like receptor (TLR)/IL1R superfamily. IL-33/ST2L then forms a comple x with the ubiquitously expressed IL-1R accessory protein (IL-1RAcP) [4-6]. Sig- nal\n",
      "Answer:\u001b[32m False\u001b[0m\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Predict if the two chunks are related by a citation. Consider all possible ways in which a citation could occur, such as direct quotes, paraphrasing, or referring to the same ideas or data. Don't be afraid to predict that the chunks are related by a citation. If you're not sure, it's better to predict that they are related.\n",
      "\n",
      "---\n",
      "\n",
      "Follow the following format.\n",
      "\n",
      "Query Chunk: Query chunk to compare to the candidate chunk.\n",
      "Candidate Chunk: Candidate chunk to compare to the query chunk.\n",
      "Answer: either True or False\n",
      "\n",
      "---\n",
      "\n",
      "Query Chunk: Sigma1, respectively. IKis a(K×K)identity matrix and τis a scalar inverse variance of /epsilon1. In this PPCA model, a complete log-likelihood function is written as: lnp(y,x|θ)≡lnp(y,x|W,µ,τ) =−τ 2/bardbly−Wx−µ/bardbl2−1 2/bardblx/bardbl2+D 2lnτ −K+D 2ln2π, where θ≡{W,µ,τ}istheparameterset.Sincethemaximum likelihood (ML) estimation of the PPCA is identical to PCA,PPCAisanaturalextensionofPCAtoaprobabilisticmodel. WeintroducehereaBayesianestimationmethodforPPCA, which was originally proposed by Bishop (1999). Bayesianestimation obtains the posterior distribution of θandX, according to the Bayes theorem: p(θ,X|Y)∝p(Y,X|θ)p(θ). (3) p(θ)iscalledapriordistribution,whichdenotesaprioripref- erence for parameter θ. The prior distribution is a part of the model and must be deﬁned before estimation. We assume conjugate priors for τandµ, and a hierarch- ical prior for W, namely, the prior for W,p(W|τ,α),i sparameterized by a hyperparameter α∈R K. p(θ|α)≡p(µ,W,τ|α)=p(µ|τ)p(τ)K/productdisplay j=1p(wj|τ,αj), p(µ|τ)=N(\n",
      "Candidate Chunk: ule con-tains a nuclear localization signal and a homeodomain(helix-turn-helix-like motif) that can bind to heterochro-matin in the nucleus and has similar structure to theDrosophila transcription factor engrailed [2,11]. In a similar manner to which a motif found in Kaposi sar-coma herpesvirus LANA (la tency-associated nuclear antigen) attaches its viral genomes to mitotic chromo-somes, nuclear IL-33 is thought to be involved in tran-scriptional repression by binding to the H2A-H2B acidicpocket of nucleosomes and regulating chromatin com-paction by promoting nucleosome-nucleosome interac-tions [12]. However, the speci fic transcriptional targets or the biological effects of nuclear IL-33 are unclear atpresent. Both IL-1 band IL-18 are synthesized as a biologically inactive precursors and activated by caspase-1 cleavageunder pro-inflammatory conditions and it was initiallyt h o u g h tt h a tI L - 3 3u n d e r w e n ts i m i l a rp r o c e s s i n gb y Correspondence: Ashley.Miller@glasgow.ac.uk Institute\n",
      "Answer:\u001b[32m False\u001b[0m\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Predict if the two chunks are related by a citation. Consider all possible ways in which a citation could occur, such as direct quotes, paraphrasing, or referring to the same ideas or data. Don't be afraid to predict that the chunks are related by a citation. If you're not sure, it's better to predict that they are related.\n",
      "\n",
      "---\n",
      "\n",
      "Follow the following format.\n",
      "\n",
      "Query Chunk: Query chunk to compare to the candidate chunk.\n",
      "Candidate Chunk: Candidate chunk to compare to the query chunk.\n",
      "Answer: either True or False\n",
      "\n",
      "---\n",
      "\n",
      "Query Chunk: µ|µ0,(γµ0τ)−1Im), p(wj|τ,αj)=N(wj|0,(αjτ)−1Im), p(τ)=G(τ|¯τ0,γτ0). G(τ|¯τ,γτ)denotesaGammadistributionwithhyperparamet- ers¯τandγτ: G(τ|¯τ,γτ)≡(γτ¯τ−1)γτ /Gamma1(γτ)exp/bracketleftbig −γτ¯τ−1τ+(γτ−1)lnτ], where /Gamma1(·)is a Gamma function. The variables used in the above priors, γµ0,µ0,γτ0and¯τ0 aredeterministichyperparametersthatdeﬁnetheprior.Their actual values should be given before the estimation. We setγ µ0=γτ0=10−10,µ0=0and¯τ0=1, which corresponds to an almost non-informative prior. Assumingthepriorsandgivenawholedataset Y={y},the type-IIMLhyperparameter αML−IIandtheposteriordistribu- tion of the parameter, q(θ)=p(θ|Y,αML−II), are obtained by Bayesian estimation. The hierarchical prior p(W|α,τ), which is called an auto- maticrelevancedetermination(ARD)prior,hasanimportantroleinBPCA.The j-thprincipalaxis w jhasaGaussianprior, anditsvariance1 /(αjτ)iscontroledbyahyperparameter αj whichisdeterminedbytype-IIMLestimationfromthedata. WhentheEuclidiannormoftheprincipalaxis, /bardblwj/bardbl,issmall relat\n",
      "Candidate Chunk: ule con-tains a nuclear localization signal and a homeodomain(helix-turn-helix-like motif) that can bind to heterochro-matin in the nucleus and has similar structure to theDrosophila transcription factor engrailed [2,11]. In a similar manner to which a motif found in Kaposi sar-coma herpesvirus LANA (la tency-associated nuclear antigen) attaches its viral genomes to mitotic chromo-somes, nuclear IL-33 is thought to be involved in tran-scriptional repression by binding to the H2A-H2B acidicpocket of nucleosomes and regulating chromatin com-paction by promoting nucleosome-nucleosome interac-tions [12]. However, the speci fic transcriptional targets or the biological effects of nuclear IL-33 are unclear atpresent. Both IL-1 band IL-18 are synthesized as a biologically inactive precursors and activated by caspase-1 cleavageunder pro-inflammatory conditions and it was initiallyt h o u g h tt h a tI L - 3 3u n d e r w e n ts i m i l a rp r o c e s s i n gb y Correspondence: Ashley.Miller@glasgow.ac.uk Institute\n",
      "Answer:\u001b[32m False\u001b[0m\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Predict if the two chunks are related by a citation. Consider all possible ways in which a citation could occur, such as direct quotes, paraphrasing, or referring to the same ideas or data. Don't be afraid to predict that the chunks are related by a citation. If you're not sure, it's better to predict that they are related.\n",
      "\n",
      "---\n",
      "\n",
      "Follow the following format.\n",
      "\n",
      "Query Chunk: Query chunk to compare to the candidate chunk.\n",
      "Candidate Chunk: Candidate chunk to compare to the query chunk.\n",
      "Answer: either True or False\n",
      "\n",
      "---\n",
      "\n",
      "Query Chunk: ively to the noise variance 1 /τ, the hyperparameter αj gets large and the principal axis wjshrinks nearly to be 0. Thus, redundant principal axes are automatically supressed. 2.4 EM-like repetitive algorithm If we know the true parameter θtrue, the posterior of the missing values is given by q(Ymiss)=p(Ymiss|Yobs,θtrue), which produces equivalent estimation to the PC regression. Here, p(Ymiss|Yobs,θtrue)is obtained by marginalizing the likelihood (3) with respect to the observed variables Yobs. If we have the parameter posterior q(θ)instead of the true parameter, the posterior of the missing values is given by q(Ymiss)=/integraldisplay dθq(θ)p(Ymiss|Yobs,θ), whichcorrespondstotheBayesianPCregression.Sincewedo notknowthetrueparameternaturally,weconducttheBPCA.Althoughtheparameterposterior q(θ)canbeeasilyobtained by the Bayesian estimation when a complete data set Yis available, we assume that only a part of Y,Y obs, is observed 2090Downloaded from https://academic.oup.com/bioinformatics/article/19/16/2088\n",
      "Candidate Chunk: ofblocking anti-ST2 antibodies or ST2-Ig fusion protein inhibited Th2 cytokine production in vivo , eosinophilic pulmonary inflammation and a irways hyper-responsive- ness [50]. At present, the role of IL-33/ST2 in studies using ST2-deficient mice is unclear as these mice are notprotected in the ovalbumin-induced airway inflammationmodel but have attenuated inflammation in a short-termpriming model of asthma. Furthermore, there is also anexacerbation of disease in wild-type or Rag-1 -/-mice that had undergone adoptive transfer of ST2-/-DO11.10 Th2 cells [24,51,52]. In order to clarify the role of IL-33/ST2in lung inflammation, several groups have generatedmice deficient in IL-33. Oboki and co-workers demon-strated that 2 sensitizations of IL-33 -/-mice with ovalbu- m i ne m u l s i f i e di na l u ms h o w e da t t e n u a t e de o s i n o p h i land lymphocyte recruitment to the lung, airway hyper-responsiveness and inflammation [19]. A similar study by Louten and colleagues has also shown that endogenou\n",
      "Answer:\u001b[32m False\u001b[0m\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Predict if the two chunks are related by a citation. Consider all possible ways in which a citation could occur, such as direct quotes, paraphrasing, or referring to the same ideas or data. Don't be afraid to predict that the chunks are related by a citation. If you're not sure, it's better to predict that they are related.\n",
      "\n",
      "---\n",
      "\n",
      "Follow the following format.\n",
      "\n",
      "Query Chunk: Query chunk to compare to the candidate chunk.\n",
      "Candidate Chunk: Candidate chunk to compare to the query chunk.\n",
      "Answer: either True or False\n",
      "\n",
      "---\n",
      "\n",
      "Query Chunk: /242445 by guest on 05 April 2024 Bayesian missing value estimation andtherest Ymissismissing.Inthatsituation,itisrequiredto obtain q(θ)andq(Ymiss)simultaneously. We use a variational Bayes (VB) algorithm (Attias, 1999), in order to execute Bayesian estimation for both model para-meterθandmissingvalues Y miss.AlthoughtheVBalgorithm resembles the EM algorithm that obtains ML estimators forθandY miss, it obtains the posterior distributions for θand Ymiss,q(θ)andq(Ymiss), by a repetitive algorithm. The VB algorithm is implemented as follows: (a) the pos- terior distribution of missing values, q(Ymiss), is initialized byimputingeachofthemissingvaluestogene-wiseaverage;(b)theposteriordistributionoftheparameter θ,q(θ),isestim- ated using the observed data Y obsand the current posterior distributionofmissingvalues, q(Ymiss);(c)theposteriordis- tribution of the missing values, q(Ymiss), is estimated using the current q(θ); (d) the hyperparameter αis updated using both of the current q(θ)and the current q(Ymiss);\n",
      "Candidate Chunk: ofblocking anti-ST2 antibodies or ST2-Ig fusion protein inhibited Th2 cytokine production in vivo , eosinophilic pulmonary inflammation and a irways hyper-responsive- ness [50]. At present, the role of IL-33/ST2 in studies using ST2-deficient mice is unclear as these mice are notprotected in the ovalbumin-induced airway inflammationmodel but have attenuated inflammation in a short-termpriming model of asthma. Furthermore, there is also anexacerbation of disease in wild-type or Rag-1 -/-mice that had undergone adoptive transfer of ST2-/-DO11.10 Th2 cells [24,51,52]. In order to clarify the role of IL-33/ST2in lung inflammation, several groups have generatedmice deficient in IL-33. Oboki and co-workers demon-strated that 2 sensitizations of IL-33 -/-mice with ovalbu- m i ne m u l s i f i e di na l u ms h o w e da t t e n u a t e de o s i n o p h i land lymphocyte recruitment to the lung, airway hyper-responsiveness and inflammation [19]. A similar study by Louten and colleagues has also shown that endogenou\n",
      "Answer:\u001b[32m False\u001b[0m\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "llm.inspect_history(n=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}