optimizer_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# In-context learning for Citation Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import dspy\n",
    "import pandas as pd\n",
    "import ast\n",
    "import numpy as np\n",
    "import os\n",
    "from numpy.linalg import norm\n",
    "from tqdm import tqdm\n",
    "from pathlib import Path\n",
    "# from operator import add\n",
    "from PyPDF2 import PdfReader\n",
    "from openai import OpenAI\n",
    "from dspy.evaluate import Evaluate\n",
    "# import random\n",
    "# from dotenv import load_dotenv\n",
    "\n",
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get the test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "query_candidate_data = pd.read_csv('darwin/test.qrel.cid', sep=' ', header=None, names=['query', 'candidate', 'bool'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len(query_papers): 115\n",
      "len(candidate_papers): 637\n"
     ]
    }
   ],
   "source": [
    "query_dir = 'darwin/query_papers'\n",
    "candidate_dir = 'darwin/candidate_papers'\n",
    "retrieved_dir = 'darwin/retrieved-papers'\n",
    "\n",
    "with open('darwin/qpaper_to_emb', 'r') as f:\n",
    "    query_papers = [line.strip() for line in f]\n",
    "\n",
    "with open('darwin/cpaper_to_emb', 'r') as f:\n",
    "    candidate_papers = [line.strip() for line in f]\n",
    "\n",
    "print(f'len(query_papers): {len(query_papers)}')\n",
    "print(f'len(candidate_papers): {len(candidate_papers)}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     query candidate  bool\n",
      "0  3498240   1824499     1\n",
      "1  3498240  53645322     0\n",
      "2  3498240   1915951     0\n",
      "3  3498240   3048298     0\n",
      "4  3498240   3627503     0\n",
      "Number of query candidate pairs with valid files: 651\n"
     ]
    }
   ],
   "source": [
    "counter_4 = 0\n",
    "valid_rows = pd.DataFrame()\n",
    "# Iterate over the rows of the data\n",
    "for _, row in query_candidate_data.iterrows():\n",
    "    query_file = os.path.join(query_dir, str(row['query']) + '.pdf')\n",
    "    candidate_file = os.path.join(candidate_dir, str(row['candidate']) + '.pdf')\n",
    "\n",
    "    # Check if both files exist\n",
    "    if os.path.isfile(query_file) and os.path.isfile(candidate_file):\n",
    "        # If both files exist, append the row to valid_rows\n",
    "        valid_rows = valid_rows._append(row)\n",
    "        \n",
    "# Reset the index of valid_rows\n",
    "valid_rows.reset_index(drop=True, inplace=True)\n",
    "print(valid_rows.head())\n",
    "print(f'Number of query candidate pairs with valid files: {len(valid_rows)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = [{\"query_file\": query_file, \"candidate_file\": candidate_file, \"cites\": bool(bool_)} for query_file, candidate_file, bool_ in zip(valid_rows['query'], valid_rows['candidate'], valid_rows['bool'])]\n",
    "data = [dspy.Example(**x).with_inputs('query_file', 'candidate_file') for x in data]\n",
    "\n",
    "def split_data(data, split_ratio, seed=42):\n",
    "    np.random.seed(seed)\n",
    "    indices = np.random.permutation(len(data))\n",
    "    split_index = int(split_ratio * len(data))\n",
    "    train_indices = indices[:split_index]\n",
    "    test_indices = indices[split_index:]\n",
    "    trainset = [data[i] for i in train_indices]\n",
    "    testset = [data[i] for i in test_indices]\n",
    "    return trainset, testset\n",
    "\n",
    "# trainset, testset = split_data(data, 0)\n",
    "trainset = data\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Produce the dataset where dspy will retrieve from\n",
    "Each sample has the following format. \"Query Chunk: ...\\n Candidate Chunk: ...\\n Answer: ...\\n  \" "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_papers = []\n",
    "test_paper_paths = []\n",
    "r_paper_paths = []\n",
    "r_papers = []\n",
    "with open('darwin/link-recorder-final-1', 'r') as f:\n",
    "    for line in f:\n",
    "        temp = line.split('\\t')\n",
    "        tpaper = temp[0].strip()\n",
    "        rpaper = temp[1].strip()\n",
    "        # Check if tpaper exists in query_papers or candidate_papers\n",
    "        if tpaper in query_papers:\n",
    "            # print(f'tpaper {tpaper} exists in query_papers')\n",
    "            tpaper_path = os.path.join(query_dir, tpaper + '.pdf')\n",
    "        elif tpaper in candidate_papers:\n",
    "            tpaper_path = os.path.join(candidate_dir, tpaper + '.pdf')\n",
    "        else:\n",
    "            # print(f'tpaper {tpaper} does not exist in query_papers or candidate_papers')\n",
    "            continue\n",
    "        r_paper_path = os.path.join(retrieved_dir, rpaper + '.pdf')\n",
    "        \n",
    "        test_papers.append(tpaper)\n",
    "        test_paper_paths.append(tpaper_path)\n",
    "        r_papers.append(rpaper)\n",
    "        r_paper_paths.append(r_paper_path)\n",
    "test_retrieved_data = pd.DataFrame({'tpaper': test_papers, 'rpaper': r_papers, 'tpaper_path': test_paper_paths, 'rpaper_path': r_paper_paths})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tpaper</th>\n",
       "      <th>rpaper</th>\n",
       "      <th>tpaper_path</th>\n",
       "      <th>rpaper_path</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1845461</td>\n",
       "      <td>58230747</td>\n",
       "      <td>darwin/candidate_papers/1845461.pdf</td>\n",
       "      <td>darwin/retrieved-papers/58230747.pdf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>18542228</td>\n",
       "      <td>18542228</td>\n",
       "      <td>darwin/candidate_papers/18542228.pdf</td>\n",
       "      <td>darwin/retrieved-papers/18542228.pdf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>52943195</td>\n",
       "      <td>222420988</td>\n",
       "      <td>darwin/candidate_papers/52943195.pdf</td>\n",
       "      <td>darwin/retrieved-papers/222420988.pdf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>17153635</td>\n",
       "      <td>244273135</td>\n",
       "      <td>darwin/candidate_papers/17153635.pdf</td>\n",
       "      <td>darwin/retrieved-papers/244273135.pdf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>24269741</td>\n",
       "      <td>79055678</td>\n",
       "      <td>darwin/candidate_papers/24269741.pdf</td>\n",
       "      <td>darwin/retrieved-papers/79055678.pdf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>232</th>\n",
       "      <td>33508063</td>\n",
       "      <td>187967027</td>\n",
       "      <td>darwin/candidate_papers/33508063.pdf</td>\n",
       "      <td>darwin/retrieved-papers/187967027.pdf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>233</th>\n",
       "      <td>5280973</td>\n",
       "      <td>17823317</td>\n",
       "      <td>darwin/candidate_papers/5280973.pdf</td>\n",
       "      <td>darwin/retrieved-papers/17823317.pdf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>234</th>\n",
       "      <td>7103019</td>\n",
       "      <td>222209029</td>\n",
       "      <td>darwin/candidate_papers/7103019.pdf</td>\n",
       "      <td>darwin/retrieved-papers/222209029.pdf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235</th>\n",
       "      <td>58604136</td>\n",
       "      <td>207945705</td>\n",
       "      <td>darwin/candidate_papers/58604136.pdf</td>\n",
       "      <td>darwin/retrieved-papers/207945705.pdf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>236</th>\n",
       "      <td>2461588</td>\n",
       "      <td>158453633</td>\n",
       "      <td>darwin/candidate_papers/2461588.pdf</td>\n",
       "      <td>darwin/retrieved-papers/158453633.pdf</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>237 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       tpaper     rpaper                           tpaper_path  \\\n",
       "0     1845461   58230747   darwin/candidate_papers/1845461.pdf   \n",
       "1    18542228   18542228  darwin/candidate_papers/18542228.pdf   \n",
       "2    52943195  222420988  darwin/candidate_papers/52943195.pdf   \n",
       "3    17153635  244273135  darwin/candidate_papers/17153635.pdf   \n",
       "4    24269741   79055678  darwin/candidate_papers/24269741.pdf   \n",
       "..        ...        ...                                   ...   \n",
       "232  33508063  187967027  darwin/candidate_papers/33508063.pdf   \n",
       "233   5280973   17823317   darwin/candidate_papers/5280973.pdf   \n",
       "234   7103019  222209029   darwin/candidate_papers/7103019.pdf   \n",
       "235  58604136  207945705  darwin/candidate_papers/58604136.pdf   \n",
       "236   2461588  158453633   darwin/candidate_papers/2461588.pdf   \n",
       "\n",
       "                               rpaper_path  \n",
       "0     darwin/retrieved-papers/58230747.pdf  \n",
       "1     darwin/retrieved-papers/18542228.pdf  \n",
       "2    darwin/retrieved-papers/222420988.pdf  \n",
       "3    darwin/retrieved-papers/244273135.pdf  \n",
       "4     darwin/retrieved-papers/79055678.pdf  \n",
       "..                                     ...  \n",
       "232  darwin/retrieved-papers/187967027.pdf  \n",
       "233   darwin/retrieved-papers/17823317.pdf  \n",
       "234  darwin/retrieved-papers/222209029.pdf  \n",
       "235  darwin/retrieved-papers/207945705.pdf  \n",
       "236  darwin/retrieved-papers/158453633.pdf  \n",
       "\n",
       "[237 rows x 4 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_retrieved_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     tpaper     rpaper                           tpaper_path  \\\n",
      "0   1845461   58230747   darwin/candidate_papers/1845461.pdf   \n",
      "1  18542228   18542228  darwin/candidate_papers/18542228.pdf   \n",
      "2  52943195  222420988  darwin/candidate_papers/52943195.pdf   \n",
      "3  17153635  244273135  darwin/candidate_papers/17153635.pdf   \n",
      "4  24269741   79055678  darwin/candidate_papers/24269741.pdf   \n",
      "\n",
      "                             rpaper_path  \n",
      "0   darwin/retrieved-papers/58230747.pdf  \n",
      "1   darwin/retrieved-papers/18542228.pdf  \n",
      "2  darwin/retrieved-papers/222420988.pdf  \n",
      "3  darwin/retrieved-papers/244273135.pdf  \n",
      "4   darwin/retrieved-papers/79055678.pdf  \n",
      "Number of query candidate pairs with valid files: 237\n"
     ]
    }
   ],
   "source": [
    "valid_rows_retrieved = pd.DataFrame()\n",
    "retrieved_dir = 'darwin/retrieved-papers'\n",
    "for _, row in test_retrieved_data.iterrows():\n",
    "    # # Check if both files exist\n",
    "    if os.path.isfile(row['tpaper_path']) and os.path.isfile(row['rpaper_path']):\n",
    "        # If both files exist, append the row to valid_rows\n",
    "        valid_rows_retrieved = valid_rows_retrieved._append(row)\n",
    "valid_rows_retrieved.reset_index(drop=True, inplace=True)\n",
    "print(valid_rows_retrieved.head())\n",
    "print(f'Number of query candidate pairs with valid files: {len(valid_rows_retrieved)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tpaper</th>\n",
       "      <th>rpaper</th>\n",
       "      <th>tpaper_path</th>\n",
       "      <th>rpaper_path</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1845461</td>\n",
       "      <td>58230747</td>\n",
       "      <td>darwin/candidate_papers/1845461.pdf</td>\n",
       "      <td>darwin/retrieved-papers/58230747.pdf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>18542228</td>\n",
       "      <td>18542228</td>\n",
       "      <td>darwin/candidate_papers/18542228.pdf</td>\n",
       "      <td>darwin/retrieved-papers/18542228.pdf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>52943195</td>\n",
       "      <td>222420988</td>\n",
       "      <td>darwin/candidate_papers/52943195.pdf</td>\n",
       "      <td>darwin/retrieved-papers/222420988.pdf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>17153635</td>\n",
       "      <td>244273135</td>\n",
       "      <td>darwin/candidate_papers/17153635.pdf</td>\n",
       "      <td>darwin/retrieved-papers/244273135.pdf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>24269741</td>\n",
       "      <td>79055678</td>\n",
       "      <td>darwin/candidate_papers/24269741.pdf</td>\n",
       "      <td>darwin/retrieved-papers/79055678.pdf</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     tpaper     rpaper                           tpaper_path  \\\n",
       "0   1845461   58230747   darwin/candidate_papers/1845461.pdf   \n",
       "1  18542228   18542228  darwin/candidate_papers/18542228.pdf   \n",
       "2  52943195  222420988  darwin/candidate_papers/52943195.pdf   \n",
       "3  17153635  244273135  darwin/candidate_papers/17153635.pdf   \n",
       "4  24269741   79055678  darwin/candidate_papers/24269741.pdf   \n",
       "\n",
       "                             rpaper_path  \n",
       "0   darwin/retrieved-papers/58230747.pdf  \n",
       "1   darwin/retrieved-papers/18542228.pdf  \n",
       "2  darwin/retrieved-papers/222420988.pdf  \n",
       "3  darwin/retrieved-papers/244273135.pdf  \n",
       "4   darwin/retrieved-papers/79055678.pdf  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_rows_retrieved.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of query candidate pairs in dspy retrieval set: 900\n",
      "     query  candidate  label                           query_path  \\\n",
      "0  1624655   55801602   True  darwin/candidate_papers/1624655.pdf   \n",
      "1  1624655   55801602  False  darwin/candidate_papers/1624655.pdf   \n",
      "2  1624655   92798697  False  darwin/candidate_papers/1624655.pdf   \n",
      "3  1624655  253224152  False  darwin/candidate_papers/1624655.pdf   \n",
      "4  1624655   10302429  False  darwin/candidate_papers/1624655.pdf   \n",
      "\n",
      "                          candidate_path  \n",
      "0   darwin/retrieved-papers/55801602.pdf  \n",
      "1   darwin/retrieved-papers/55801602.pdf  \n",
      "2   darwin/retrieved-papers/92798697.pdf  \n",
      "3  darwin/retrieved-papers/253224152.pdf  \n",
      "4   darwin/retrieved-papers/10302429.pdf  \n"
     ]
    }
   ],
   "source": [
    "# Randoly select 100 to set up the retrieval dataset\n",
    "dspy_r_set = pd.DataFrame(columns=['query', 'candidate', 'label'])\n",
    "\n",
    "# valid_r_papers = valid_rows_retrieved['rpaper'].to_numpy()\n",
    "sampled_df = valid_rows_retrieved.sample(n=100, random_state=42)\n",
    "\n",
    "queries = []\n",
    "query_paths = []\n",
    "candidates = []\n",
    "candidate_paths = []\n",
    "labels = []\n",
    "for _, row in sampled_df.iterrows():\n",
    "    test_set_paper = row['tpaper']\n",
    "    retrieved_paper = row['rpaper']\n",
    "    queries.append(test_set_paper)\n",
    "    candidates.append(retrieved_paper)\n",
    "    query_paths.append(row['tpaper_path'])\n",
    "    candidate_paths.append(row['rpaper_path'])\n",
    "    labels.append(True)\n",
    "    assert os.path.join(retrieved_dir, retrieved_paper + '.pdf') == row['rpaper_path'], f'{os.path.join(retrieved_dir, retrieved_paper + \".pdf\")} != {row[\"rpaper_path\"]}'\n",
    " \n",
    "    # Get 8 negative samples for one positive sample as in the SPECTER svm experiment.\n",
    "    neg_papers = valid_rows_retrieved.sample(n=8)\n",
    "    rpapers = neg_papers['rpaper'].values\n",
    "    rpaper_paths = neg_papers['rpaper_path'].values\n",
    "    for rpaper_path, rpaper in zip(rpaper_paths, rpapers):\n",
    "        queries.append(test_set_paper)\n",
    "        candidates.append(rpaper)\n",
    "        labels.append(False)\n",
    "        query_paths.append(row['tpaper_path'])\n",
    "        candidate_paths.append(rpaper_path)\n",
    "    \n",
    "dspy_r_set = pd.DataFrame({'query': queries, 'candidate': candidates, 'label': labels, 'query_path': query_paths, 'candidate_path': candidate_paths})\n",
    "\n",
    "print(f'Number of query candidate pairs in dspy retrieval set: {len(dspy_r_set)}')\n",
    "print(dspy_r_set.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Chunker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "llm = dspy.OpenAI(model=\"gpt-3.5-turbo\")\n",
    "dspy.settings.configure(lm=llm, rm=None)\n",
    "\n",
    "client = OpenAI(\n",
    "    # this is also the default, it can be omitted\n",
    "    api_key=os.environ['OPENAI_API_KEY'],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Chunker:\n",
    "    def __init__(self, context_window=3000, max_windows=5):\n",
    "        self.context_window = context_window\n",
    "        self.max_windows = max_windows\n",
    "        self.window_overlap = 0.02\n",
    "\n",
    "    def __call__(self, paper):\n",
    "        snippet_idx = 0\n",
    "\n",
    "        while snippet_idx < self.max_windows and paper:\n",
    "            endpos = int(self.context_window * (1.0 + self.window_overlap))\n",
    "            snippet, paper = paper[:endpos], paper[endpos:]\n",
    "\n",
    "            next_newline_pos = snippet.rfind('\\n')\n",
    "            if paper and next_newline_pos != -1 and next_newline_pos >= self.context_window // 2:\n",
    "                paper = snippet[next_newline_pos+1:] + paper\n",
    "                snippet = snippet[:next_newline_pos]\n",
    "\n",
    "            yield snippet_idx, snippet.strip()\n",
    "            snippet_idx += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate Retrieval Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_embeddings(texts, model=\"text-embedding-3-small\", save_file=None):\n",
    "    if save_file and Path(save_file).exists():\n",
    "        with open(save_file, 'r') as f:\n",
    "            # print(f\"Loading embeddings from {save_file}\")\n",
    "            embeddings = [ast.literal_eval(line.strip()) for line in f]\n",
    "            # assert len(embeddings) == len(texts), f\"Number of embeddings does not match number of texts {len(embeddings)} != {len(texts)}\"\n",
    "        return embeddings\n",
    "        \n",
    "    try:\n",
    "        response = client.embeddings.create(input=texts, model=model)\n",
    "        embeddings = [embedding.embedding for embedding in response.data]\n",
    "        assert len(embeddings) == len(texts), \"Number of embeddings does not match number of texts\"\n",
    "        if save_file: # Save the embeddings to a file\n",
    "            with open(save_file, 'w') as f:\n",
    "                # print(f\"Saving embeddings to {save_file}\")\n",
    "                for embedding in embeddings:\n",
    "                    f.write(str(embedding) + '\\n')\n",
    "        return embeddings\n",
    "    except Exception as e:\n",
    "        print(\"Error during API call:\", e)\n",
    "        return []\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_most_similar_chunk_emb(query_embedding, candidate_embeddings, candidate_chunks):\n",
    "    similarities = np.dot(candidate_embeddings, query_embedding) / (norm(candidate_embeddings, axis=1) * norm(query_embedding))\n",
    "    most_similar_idx = np.argmax(similarities)\n",
    "    return candidate_chunks[most_similar_idx], candidate_embeddings[most_similar_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_most_similar_emb_idx(query_embedding, candidate_embeddings):\n",
    "    similarities = np.dot(candidate_embeddings, query_embedding) / (norm(candidate_embeddings, axis=1) * norm(query_embedding))\n",
    "    most_similar_idx = np.argmax(similarities)\n",
    "    return most_similar_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "dspy_r_set.to_csv('darwin/dspy_r_set.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|▏         | 20/900 [00:18<10:55,  1.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error reading candidate file darwin/retrieved-papers/49409974.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  8%|▊         | 73/900 [01:21<13:59,  1.01s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error reading candidate file darwin/retrieved-papers/199472931.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|▉         | 86/900 [01:39<14:01,  1.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error reading candidate file darwin/retrieved-papers/230670622.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|▉         | 89/900 [01:42<12:34,  1.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error during API call: Error code: 400 - {'error': {'message': \"'$.input' is invalid. Please check the API reference: https://platform.openai.com/docs/api-reference.\", 'type': 'invalid_request_error', 'param': None, 'code': None}}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 13%|█▎        | 116/900 [02:09<07:40,  1.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error reading candidate file darwin/retrieved-papers/230670622.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14%|█▍        | 125/900 [02:19<11:30,  1.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error reading candidate file darwin/retrieved-papers/230670622.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|█▍        | 133/900 [02:26<10:17,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error during API call: Error code: 400 - {'error': {'message': \"'$.input' is invalid. Please check the API reference: https://platform.openai.com/docs/api-reference.\", 'type': 'invalid_request_error', 'param': None, 'code': None}}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 19%|█▉        | 174/900 [03:00<12:14,  1.01s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error during API call: Error code: 400 - {'error': {'message': \"'$.input' is invalid. Please check the API reference: https://platform.openai.com/docs/api-reference.\", 'type': 'invalid_request_error', 'param': None, 'code': None}}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|██▊       | 249/900 [04:31<05:23,  2.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error reading candidate file darwin/retrieved-papers/1511942.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|██▉       | 266/900 [04:48<10:32,  1.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error reading candidate file darwin/retrieved-papers/199472931.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32%|███▏      | 287/900 [05:31<27:59,  2.74s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error reading candidate file darwin/retrieved-papers/199472931.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32%|███▏      | 288/900 [05:34<27:54,  2.74s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error during API call: Error code: 400 - {'error': {'message': \"'$.input' is invalid. Please check the API reference: https://platform.openai.com/docs/api-reference.\", 'type': 'invalid_request_error', 'param': None, 'code': None}}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34%|███▎      | 303/900 [05:55<10:09,  1.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error during API call: Error code: 400 - {'error': {'message': \"'$.input' is invalid. Please check the API reference: https://platform.openai.com/docs/api-reference.\", 'type': 'invalid_request_error', 'param': None, 'code': None}}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37%|███▋      | 332/900 [07:26<43:24,  4.59s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error reading candidate file darwin/retrieved-papers/199472931.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▌    | 495/900 [10:30<07:48,  1.16s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/PyPDF2/_cmap.py:142: PdfReadWarning: Advanced encoding /SymbolSetEncoding not implemented yet\n",
      "  warnings.warn(\n",
      " 62%|██████▏   | 558/900 [11:50<07:32,  1.32s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error reading candidate file darwin/retrieved-papers/1511942.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 622/900 [15:30<10:25,  2.25s/it]  /Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/PyPDF2/_cmap.py:142: PdfReadWarning: Advanced encoding /90ms-RKSJ-H not implemented yet\n",
      "  warnings.warn(\n",
      " 72%|███████▏  | 649/900 [16:17<06:47,  1.62s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error reading candidate file darwin/retrieved-papers/49409974.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72%|███████▏  | 652/900 [16:22<06:41,  1.62s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error reading candidate file darwin/retrieved-papers/49409974.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76%|███████▋  | 688/900 [17:15<02:31,  1.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error reading candidate file darwin/retrieved-papers/230670622.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|████████▍ | 759/900 [22:03<03:32,  1.50s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error reading candidate file darwin/retrieved-papers/49409974.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88%|████████▊ | 791/900 [22:51<02:45,  1.52s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error reading candidate file darwin/retrieved-papers/199472931.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 820/900 [23:23<01:57,  1.47s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error reading candidate file darwin/retrieved-papers/1511942.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|█████████▎| 833/900 [24:03<01:37,  1.45s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error reading candidate file darwin/retrieved-papers/1511942.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 94%|█████████▎| 843/900 [24:09<00:31,  1.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error reading candidate file darwin/retrieved-papers/1511942.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 900/900 [29:09<00:00,  1.94s/it]\n"
     ]
    }
   ],
   "source": [
    "# produce retrieval set\n",
    "chunk = Chunker(context_window=1000, max_windows=15)\n",
    "dspy_r_emb = []\n",
    "dspy_r_text = []\n",
    "dspy_r_emb_concat = []\n",
    "for _, row in tqdm(dspy_r_set.iterrows(), total=len(dspy_r_set)):\n",
    "    # Get the text from the pdfs\n",
    "    query_file = row['query']\n",
    "    query_file_path = row['query_path']\n",
    "    try:\n",
    "        # print(f'reading query file {row[\"query_path\"]}')\n",
    "        query_pdf = PdfReader(query_file_path)\n",
    "    except:\n",
    "        # print(f'error reading query file {row[\"query_path\"]}')\n",
    "        continue\n",
    "    query_text = \"\"\n",
    "    for page in query_pdf.pages:\n",
    "        page_text = page.extract_text()\n",
    "        if page_text:\n",
    "            query_text += page_text + \" \"  # Adding space to separate text between pages\n",
    "    query_text = query_text.replace(\"\\n\", \" \")\n",
    "\n",
    "    candidate_file = row['candidate']\n",
    "    try:\n",
    "        # print(f'reading candidate file {row[\"candidate_path\"]}')\n",
    "        candidate_pdf = PdfReader(row['candidate_path'])\n",
    "        \n",
    "    except:\n",
    "        print(f'Error reading candidate file {row[\"candidate_path\"]}')\n",
    "        continue\n",
    "    candidate_text = \"\"\n",
    "    for page in candidate_pdf.pages:\n",
    "        page_text = page.extract_text()\n",
    "        if page_text:\n",
    "            candidate_text += page_text + \" \"\n",
    "    candidate_text = candidate_text.replace(\"\\n\", \" \")\n",
    "\n",
    "    # for each chunk in the paper\n",
    "    query_chunks = [snippet for _, snippet in chunk(query_text)]\n",
    "    candidate_chunks = [snippet for _, snippet in chunk(candidate_text)]\n",
    "    # print(f'len(query_chunks): {len(query_chunks)} len(candidate_chunks): {len(candidate_chunks)}')\n",
    "\n",
    "    # Create embeddings for the chunks\n",
    "    candidate_embeddings = get_embeddings(candidate_chunks, save_file=f'new_embeddings/candidate_{candidate_file}.emb')\n",
    "    query_embeddings = get_embeddings(query_chunks, save_file=f'new_embeddings/query_{query_file}.emb')\n",
    "    assert len(candidate_embeddings) == len(candidate_chunks), f\"Number of embeddings does not match number of texts {len(candidate_embeddings)} != {len(candidate_chunks)}\"\n",
    "\n",
    "    for snippet, query_embedding in zip(query_chunks, query_embeddings):\n",
    "        # Get the candidate chunk that is most similar to the snippet\n",
    "        if len(candidate_embeddings) == 0 or len(query_embedding) == 0:\n",
    "            continue\n",
    "        if len(candidate_embeddings) != len(candidate_chunks):\n",
    "            print('Error')\n",
    "            print(f'len(candidate_embeddings): {len(candidate_embeddings)}')\n",
    "            print(f'len(candidate_chunks): {len(candidate_chunks)}')\n",
    "            continue\n",
    "        # assert(len(candidate_embeddings) == len(candidate_chunks))\n",
    "        candidate_chunk, c_emb = get_most_similar_chunk_emb(query_embedding, candidate_embeddings, candidate_chunks)\n",
    "        dspy_r_emb.append((query_embedding, c_emb, row['label']))\n",
    "        dspy_r_text.append((snippet, candidate_chunk, row['label']))\n",
    "        dspy_r_emb_concat.append(query_embedding + c_emb)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DSPy Module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PredictCitationWithRetrieval(dspy.Signature):\n",
    "    __doc__ = \"\"\"Predict if the two chunks are related by a citation. Consider all possible ways in which a citation could occur, such as direct quotes, paraphrasing, or referring to the same ideas or data. Don't be afraid to predict that the chunks are related by a citation. If you're not sure, it's better to predict that they are related.\"\"\"   \n",
    "    query_chunk: str = dspy.InputField(desc='Query chunk to compare to the candidate chunk.')\n",
    "    candidate_chunk: str = dspy.InputField(desc='Candidate chunk to compare to the query chunk.')\n",
    "    answer: bool = dspy.OutputField(desc=\"either True or False\", prefix=\"Answer:\")\n",
    "    example: str = dspy.InputField(desc=\"A good example to learn from.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "class PredictCitationRetrieveAndResolve(dspy.Module):\n",
    "    def __init__(self, context_window=3000, max_windows=5, resolve_function=any,\n",
    "                 candidate_folder='darwin/candidate_papers/', \n",
    "                 query_folder='darwin/query_papers',\n",
    "                 reset_embedding=False):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.chunk = Chunker(context_window=context_window, max_windows=max_windows)\n",
    "        # self.predict = dspy.TypedPredictor(PredictCitation)\n",
    "        # self.predict = dspy.TypedChainOfThought(PredictCitation)\n",
    "        self.predict = dspy.ChainOfThought(PredictCitationWithRetrieval)\n",
    "        # self.predict = dspy.Predict(PredictCitationWithRetrieval)\n",
    "        self.resolve_function = resolve_function\n",
    "        self.query_folder = query_folder\n",
    "        self.candidate_folder = candidate_folder\n",
    "        os.makedirs('embeddings', exist_ok=True)\n",
    "        os.makedirs('new_embeddings', exist_ok=True)\n",
    "        if reset_embedding:\n",
    "            for emb_file in os.listdir('embeddings'):\n",
    "                os.remove(f'embeddings/{emb_file}')\n",
    "            for emb_file in os.listdir('new_embeddings'):\n",
    "                os.remove(f'new_embeddings/{emb_file}')\n",
    "\n",
    "    def forward(self, query_file, candidate_file):\n",
    "        predictions = []     \n",
    "        # Get the text from the pdfs\n",
    "        query_pdf = PdfReader(f'{self.query_folder}/{query_file}.pdf')\n",
    "        query_text = \"\"\n",
    "        for page in query_pdf.pages:\n",
    "            page_text = page.extract_text()\n",
    "            if page_text:\n",
    "                query_text += page_text + \" \"  # Adding space to separate text between pages\n",
    "        query_text = query_text.replace(\"\\n\", \" \")\n",
    "        \n",
    "        candidate_pdf = PdfReader(f'{self.candidate_folder}/{candidate_file}.pdf')\n",
    "        candidate_text = \"\"\n",
    "        for page in candidate_pdf.pages:\n",
    "            page_text = page.extract_text()\n",
    "            if page_text:\n",
    "                candidate_text += page_text + \" \"\n",
    "        candidate_text = candidate_text.replace(\"\\n\", \" \")\n",
    "        \n",
    "        # for each chunk in the paper\n",
    "        query_chunks = [snippet for _, snippet in self.chunk(query_text)]\n",
    "        candidate_chunks = [snippet for _, snippet in self.chunk(candidate_text)]\n",
    "        \n",
    "        # Create embeddings for the chunks\n",
    "        candidate_embeddings = get_embeddings(candidate_chunks, save_file=f'embeddings/candidate_{candidate_file}.emb')\n",
    "        query_embeddings = get_embeddings(query_chunks, save_file=f'embeddings/query_{query_file}.emb')\n",
    "        \n",
    "        for snippet, query_embedding in zip(query_chunks, query_embeddings):\n",
    "            # Get the candidate chunk that is most similar to the snippet\n",
    "            candidate_chunk, candidate_chunk_emb = get_most_similar_chunk_emb(query_embedding, candidate_embeddings, candidate_chunks)\n",
    "            original_emb_concat = query_embedding + candidate_chunk_emb\n",
    "            context_idx = get_most_similar_emb_idx(original_emb_concat, dspy_r_emb_concat)\n",
    "            context_text = dspy_r_text[context_idx]\n",
    "            if context_text[2]:\n",
    "                context_answer = \"True\"\n",
    "            else:\n",
    "                context_answer = \"False\"\n",
    "            example = f\"Query Chunk: {context_text[0]}\\nCandidate Chunk: {context_text[1]}\\nAnswer: {context_answer}\\n\"\n",
    "            prediction = self.predict(query_chunk=snippet, candidate_chunk=candidate_chunk, example=example)\n",
    "            # print(prediction)\n",
    "            predictions.append(prediction.answer=='True')\n",
    "            if prediction.answer == 'True':\n",
    "                break\n",
    "        return dspy.Prediction(example=example, predictions=predictions, resolved=self.resolve_function(predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def metric(example, result):\n",
    "    '''Match metric'''\n",
    "    return 1 if example.cites == result.resolved else 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_chunking_retrieval = PredictCitationRetrieveAndResolve(max_windows=15, context_window=1000, reset_embedding=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Example({'query_file': 1323414, 'candidate_file': '3324808'}) (input_keys=None)\n",
      "Example({'cites': False}) (input_keys=None)\n",
      "Prediction(\n",
      "    example='Query Chunk: ll et al., 1999a,b). Imaging protocol. The MRI protocol collected for each subject was a whole-brain, gradient-echo (spoiled gradient recalled acquisition in asteady state) T1-weighted series collected in the sagittal plane withrepetition time of 24 msec, echo time of 5 msec, two excitations, ﬂip angle of 45 °,ﬁeld of view of 24 cm, 124 slices with section thickness of 1.2 mm, no gaps, and an imaging time of 19 min. Image analysis. MR images from each individual were processed with a series of manual and automated procedures that included the followingsteps: (1) automated linear transformation (Woods et al., 1993) of theimages into a standard orientation with scaling to remove global differ-ences in head size allowing assessment of local changes in brain size ortissue density; (2) classi ﬁcation of brain images into gray matter, white matter, and CSF (Kollokian, 1996; Sowell et al., 1999b); (3) removal ofnonbrain tissue (i.e., scalp and orbits) and cerebellum from the trans-formed images; (4) automated ex\\nCandidate Chunk: three-dimensional (3D) Magnetization PreparedRapid Acquisition Gradient Echo (MPRAGE) sequence with thefollowing parameters: time repetition (TR): 2250 ms., time echo(TE): 2.6 ms., field of view (FOV): 256 6256 mm 3, slices: 176 and a voxel size of 1 6161.1 mm3. Surface reconstruction MRI-data were processed with the surface and volume pipeline of the FreeSurfer-software version 5.1.0 (http://surfer.nmr.mgh. harvard.edu) [25,26] and estimates of cortical thickness, GM- and WM- volume, cortical surface area, the 3-D local gyrificationIndex (lGI) and estimated intracranial volume (eTIV) wereobtained. The standard FreeSurfer pipeline was followed andautomatically reconstructed surfaces were inspected for accuracyand if necessary, manual interventions using FreeSurfer correctiontools were used. Pre-processing included Talairach transformation, motion correction, intensity normalization, non brain tissue removing,segmentation and tessellation of the gray and white matterboundary, automatically topology correc\\nAnswer: True\\n',\n",
      "    predictions=[True],\n",
      "    resolved=True\n",
      ")\n",
      "False\n"
     ]
    }
   ],
   "source": [
    "# get an example\n",
    "example = trainset[-2]\n",
    "example_x = example.inputs()\n",
    "example_y = example.labels()\n",
    "print(example_x)\n",
    "print(example_y)\n",
    "\n",
    "prediction = pipeline_chunking_retrieval(**example_x)\n",
    "print(prediction)\n",
    "print(example_y.cites)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "Predict if the two chunks are related by a citation. Consider all possible ways in which a citation could occur, such as direct quotes, paraphrasing, or referring to the same ideas or data. Don't be afraid to predict that the chunks are related by a citation. If you're not sure, it's better to predict that they are related.\n",
      "\n",
      "---\n",
      "\n",
      "Follow the following format.\n",
      "\n",
      "Query Chunk: Query chunk to compare to the candidate chunk.\n",
      "\n",
      "Candidate Chunk: Candidate chunk to compare to the query chunk.\n",
      "\n",
      "Example: A good example to learn from.\n",
      "\n",
      "Reasoning: Let's think step by step in order to ${produce the answer}. We ...\n",
      "\n",
      "Answer: either True or False\n",
      "\n",
      "---\n",
      "\n",
      "Query Chunk: JNERJOURNAL OF NEUROENGINEERING AND REHABILITATIONSalazar-Varas et al. Journal of NeuroEngineering and Rehabilitation (2015) 12:101 DOI 10.1186/s12984-015-0095-4 RESEARCH Open Access Analyzing EEG signals to detect unexpected obstacles during walking R. Salazar-Varas1,Á .C o s t a2,E .I á ñ e z2*,A .Ú b e d a2,E .H o r t a l2a n dJ .M .A z o r í n2 Abstract Background: When an unexpected perturbation in the environment occurs, the subsequent alertness state may cause a brain activation responding to that perturbation which can be detected and employed by a Brain-Computer Interface (BCI). In this work, the possibility of detecting a sudden obstacle appearance analyzing electroencephalographic (EEG) signals is assessed. For this purpose, different features of EEG signals are evaluated during the appearance of sudden obstacles while a subject is walking on a treadmill. The future goal is to use this procedure to detect any obstacle appearance during walking when the user is wearing a lower limb exoskeleton\n",
      "\n",
      "Candidate Chunk: 013a environment. For each subject, echo-planar images subsequently underwent steps for slice-timing correction, spatial realign-ment, and registration to standardized MNI space carried out with a population template generated from the T1 structural images using the Diffeomorphic Anatomical Registration Through Exponentiated Lie Algebra algorithm [Ashburner, 2007]. Spurious variance was reduced by regressing-out signal from white matter, cerebrospinal ﬂuid, and by voxel-speciﬁc motion correction [Sat- terthwaite et al., 2013]. Next, the images were detrended and band-pass ﬁltered (0.01–0.1 Hz) to eliminate biologi-cally non-relevant signals and the resulting output was used to calculate sample entropy [Liu et al., 2013; Richman and Moorman, 2000] at the voxel-level, as implemented in the “complexity” toolbox (LOFT Lab). Sample entropy is formally deﬁned as the negative loga- rithm of the conditional probability that if two sets of simultaneous data points (vector pairs) with length m meet similarity crite\n",
      "\n",
      "Example: Query Chunk: ll et al., 1999a,b). Imaging protocol. The MRI protocol collected for each subject was a whole-brain, gradient-echo (spoiled gradient recalled acquisition in asteady state) T1-weighted series collected in the sagittal plane withrepetition time of 24 msec, echo time of 5 msec, two excitations, ﬂip angle of 45 °,ﬁeld of view of 24 cm, 124 slices with section thickness of 1.2 mm, no gaps, and an imaging time of 19 min. Image analysis. MR images from each individual were processed with a series of manual and automated procedures that included the followingsteps: (1) automated linear transformation (Woods et al., 1993) of theimages into a standard orientation with scaling to remove global differ-ences in head size allowing assessment of local changes in brain size ortissue density; (2) classi ﬁcation of brain images into gray matter, white matter, and CSF (Kollokian, 1996; Sowell et al., 1999b); (3) removal ofnonbrain tissue (i.e., scalp and orbits) and cerebellum from the trans-formed images; (4) automated ex Candidate Chunk: three-dimensional (3D) Magnetization PreparedRapid Acquisition Gradient Echo (MPRAGE) sequence with thefollowing parameters: time repetition (TR): 2250 ms., time echo(TE): 2.6 ms., field of view (FOV): 256 6256 mm 3, slices: 176 and a voxel size of 1 6161.1 mm3. Surface reconstruction MRI-data were processed with the surface and volume pipeline of the FreeSurfer-software version 5.1.0 (http://surfer.nmr.mgh. harvard.edu) [25,26] and estimates of cortical thickness, GM- and WM- volume, cortical surface area, the 3-D local gyrificationIndex (lGI) and estimated intracranial volume (eTIV) wereobtained. The standard FreeSurfer pipeline was followed andautomatically reconstructed surfaces were inspected for accuracyand if necessary, manual interventions using FreeSurfer correctiontools were used. Pre-processing included Talairach transformation, motion correction, intensity normalization, non brain tissue removing,segmentation and tessellation of the gray and white matterboundary, automatically topology correc Answer: True\n",
      "\n",
      "Reasoning: Let's think step by step in order to determine if these two chunks are related by a citation. Both chunks discuss the processing and analysis of data collected from imaging techniques. The first chunk discusses the analysis of EEG signals to detect obstacles during walking, while the second chunk discusses the processing of MRI data for brain imaging. Both chunks involve the processing and analysis of data collected from different imaging techniques, indicating a similarity in the research methods used. Therefore, it is likely that these two chunks are related by a citation.\n",
      "\n",
      "Answer:\u001b[32m True\u001b[0m\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "llm.inspect_history()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/651 [00:00<?, ?it/s]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.15s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.42s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.93s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.16s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.60s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.43s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.42s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.21s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.43s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.47s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.17s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.01s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.44s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.62s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 0 / 1  (0.0):   0%|          | 1/651 [02:30<27:12:11, 150.66s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.25s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.49s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.44s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.53s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.65s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 8.12s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.30s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.64s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.50s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.56s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.74s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.46s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.60s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.88s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 10.29s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.97s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.66s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.20s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.66s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.82s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.70s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.26s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 2 / 3  (66.7):   0%|          | 2/651 [05:29<30:05:32, 166.92s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.58s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.71s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 3 / 4  (75.0):   1%|          | 4/651 [05:38<11:44:28, 65.33s/it] /Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.76s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.07s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 6.0 / 9  (66.7):   1%|▏         | 9/651 [06:22<3:39:49, 20.54s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t '[' was never closed (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.54s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.09s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.84s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.46s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.81s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 10.94s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.82s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.48s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.11s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 7.0 / 10  (70.0):   2%|▏         | 10/651 [07:52<7:25:34, 41.71s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.83s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.04s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.68s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.18s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.17s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.22s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.29s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.75s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 7.0 / 11  (63.6):   2%|▏         | 11/651 [09:23<10:03:39, 56.59s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.68s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 8.0 / 12  (66.7):   2%|▏         | 12/651 [09:39<7:50:12, 44.15s/it] /Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.08s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.13s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.67s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.28s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 9.0 / 13  (69.2):   2%|▏         | 13/651 [10:13<7:19:04, 41.29s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.65s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.73s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.67s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.78s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 12.0 / 16  (75.0):   2%|▏         | 16/651 [11:03<4:10:40, 23.69s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.63s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 13.0 / 17  (76.5):   2%|▏         | 16/651 [11:31<4:10:40, 23.69s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.25s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 13.0 / 18  (72.2):   3%|▎         | 18/651 [11:40<3:33:49, 20.27s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.45s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 14.0 / 19  (73.7):   3%|▎         | 18/651 [11:48<3:33:49, 20.27s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.06s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 14.0 / 21  (66.7):   3%|▎         | 20/651 [12:08<3:05:23, 17.63s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.62s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 11.06s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.94s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.09s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 15.0 / 22  (68.2):   3%|▎         | 22/651 [12:57<3:35:25, 20.55s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.21s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 15.0 / 24  (62.5):   4%|▎         | 24/651 [15:15<7:11:16, 41.27s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.70s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 16.0 / 25  (64.0):   4%|▍         | 25/651 [15:52<6:56:27, 39.92s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.10s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 16.0 / 27  (59.3):   4%|▍         | 27/651 [16:03<4:02:02, 23.27s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.97s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.65s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.63s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 17.0 / 28  (60.7):   4%|▍         | 28/651 [16:26<4:01:03, 23.22s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.66s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 17.0 / 29  (58.6):   4%|▍         | 29/651 [16:49<3:59:18, 23.08s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.03s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.23s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 18.0 / 32  (56.2):   5%|▍         | 32/651 [17:39<3:06:38, 18.09s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.69s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.57s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 18.0 / 34  (52.9):   5%|▌         | 34/651 [18:22<3:25:47, 20.01s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.91s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.13s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 18.0 / 35  (51.4):   5%|▌         | 35/651 [18:59<4:17:31, 25.08s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.77s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.83s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 20.0 / 38  (52.6):   6%|▌         | 38/651 [20:19<3:33:05, 20.86s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.74s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.36s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.28s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.12s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.72s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.11s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.79s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.88s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 21.0 / 39  (53.8):   6%|▌         | 39/651 [21:29<6:02:16, 35.52s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.28s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 23.0 / 42  (54.8):   6%|▋         | 42/651 [22:12<3:37:56, 21.47s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.50s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 24.0 / 43  (55.8):   7%|▋         | 43/651 [22:36<3:44:29, 22.15s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.11s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.52s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.19s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 25.0 / 44  (56.8):   7%|▋         | 43/651 [22:58<3:44:29, 22.15s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.59s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 25.0 / 45  (55.6):   7%|▋         | 45/651 [23:12<3:16:53, 19.49s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.61s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.79s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.80s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 8.67s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 25.0 / 46  (54.3):   7%|▋         | 46/651 [23:45<3:59:14, 23.73s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.12s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.98s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.00s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.58s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.08s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.12s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 6.81s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.61s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.41s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 26.0 / 47  (55.3):   7%|▋         | 47/651 [25:35<8:19:49, 49.65s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.34s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.14s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.87s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 28.0 / 49  (57.1):   8%|▊         | 49/651 [26:23<5:52:42, 35.15s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.90s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.04s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.59s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.14s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.17s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 31.0 / 55  (56.4):   8%|▊         | 55/651 [28:17<2:41:15, 16.23s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.53s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.11s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.00s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.99s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.54s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.24s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.46s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.47s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.56s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.38s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.57s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.71s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 32.0 / 57  (56.1):   9%|▉         | 57/651 [31:06<7:14:42, 43.91s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.72s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.32s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.77s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 32.0 / 58  (55.2):   9%|▉         | 58/651 [32:34<9:23:03, 56.97s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.23s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.99s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 32.0 / 59  (54.2):   9%|▉         | 59/651 [33:05<8:05:47, 49.24s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.40s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 32.0 / 60  (53.3):   9%|▉         | 60/651 [33:20<6:24:16, 39.01s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.55s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 33.0 / 61  (54.1):   9%|▉         | 61/651 [33:27<4:47:57, 29.28s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.50s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.56s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.57s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 33.0 / 62  (53.2):  10%|▉         | 62/651 [33:36<3:48:33, 23.28s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.19s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.18s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 8.63s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 35.0 / 64  (54.7):  10%|▉         | 64/651 [34:40<4:10:40, 25.62s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.54s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 36.0 / 66  (54.5):  10%|█         | 66/651 [36:11<5:53:31, 36.26s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 6.55s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.59s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.11s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 36.0 / 67  (53.7):  10%|█         | 67/651 [36:32<5:05:41, 31.41s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.62s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.38s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 37.0 / 68  (54.4):  10%|█         | 68/651 [38:14<8:33:19, 52.83s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.51s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.37s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 41.0 / 73  (56.2):  11%|█         | 73/651 [39:53<3:12:31, 19.99s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 7.82s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 42.0 / 74  (56.8):  11%|█▏        | 74/651 [40:24<3:42:04, 23.09s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.91s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.95s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 43.0 / 76  (56.6):  12%|█▏        | 75/651 [42:28<5:08:49, 32.17s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.19s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 43.0 / 78  (55.1):  12%|█▏        | 77/651 [42:38<5:08:46, 32.28s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.95s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 43.0 / 79  (54.4):  12%|█▏        | 79/651 [42:54<3:21:40, 21.15s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.57s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.54s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.15s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 47.0 / 86  (54.7):  13%|█▎        | 86/651 [44:50<1:57:42, 12.50s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.82s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.94s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.81s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 48.0 / 87  (55.2):  13%|█▎        | 87/651 [45:15<2:33:19, 16.31s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.33s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.87s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.08s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 50.0 / 90  (55.6):  14%|█▎        | 89/651 [47:36<5:54:43, 37.87s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t negative seek value -1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 50.0 / 90  (55.6):  14%|█▍        | 90/651 [47:41<4:23:39, 28.20s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.43s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.55s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 51.0 / 92  (55.4):  14%|█▍        | 92/651 [48:09<3:05:35, 19.92s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.02s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/PyPDF2/_cmap.py:142: PdfReadWarning: Advanced encoding /UniJIS-UCS2-H not implemented yet\n",
      "  warnings.warn(\n",
      "Average Metric: 52.0 / 93  (55.9):  14%|█▍        | 93/651 [49:14<5:12:10, 33.57s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.99s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 7.88s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.20s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 52.0 / 94  (55.3):  14%|█▍        | 94/651 [49:43<4:59:48, 32.30s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.36s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 53.0 / 95  (55.8):  15%|█▍        | 95/651 [50:21<5:14:51, 33.98s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.85s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 8.21s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 54.0 / 96  (56.2):  15%|█▍        | 96/651 [50:47<4:51:05, 31.47s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.70s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.33s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.60s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 56.0 / 100  (56.0):  15%|█▌        | 100/651 [52:43<3:39:34, 23.91s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.79s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 56.0 / 101  (55.4):  16%|█▌        | 101/651 [53:28<4:39:13, 30.46s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.64s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error during API call: Error code: 400 - {'error': {'message': \"'$.input' is invalid. Please check the API reference: https://platform.openai.com/docs/api-reference.\", 'type': 'invalid_request_error', 'param': None, 'code': None}}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 56.0 / 102  (54.9):  16%|█▌        | 102/651 [53:53<4:23:39, 28.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t shapes (0,) and (1536,) not aligned: 0 (dim 0) != 1536 (dim 0)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 57.0 / 104  (54.8):  16%|█▌        | 104/651 [54:17<3:06:19, 20.44s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.16s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.28s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 6.27s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 58.0 / 105  (55.2):  16%|█▌        | 105/651 [55:00<4:08:10, 27.27s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.52s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 59.0 / 107  (55.1):  16%|█▋        | 107/651 [55:52<3:57:13, 26.17s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.29s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 6.90s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 60.0 / 108  (55.6):  16%|█▋        | 107/651 [56:14<3:57:13, 26.17s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.22s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.26s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 60.0 / 108  (55.6):  17%|█▋        | 108/651 [56:22<4:07:20, 27.33s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.29s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 60.0 / 109  (55.0):  17%|█▋        | 109/651 [56:53<4:17:42, 28.53s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.51s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.80s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.28s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 61.0 / 111  (55.0):  17%|█▋        | 111/651 [57:24<3:06:10, 20.69s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.64s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.87s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.68s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.36s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.81s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 62.0 / 112  (55.4):  17%|█▋        | 112/651 [58:21<4:42:28, 31.45s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.67s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 63.0 / 114  (55.3):  18%|█▊        | 114/651 [58:42<3:06:59, 20.89s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.65s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 7.95s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 63.0 / 115  (54.8):  18%|█▊        | 115/651 [59:36<4:34:34, 30.74s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.24s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.20s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.22s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.80s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 6.86s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.21s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.90s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.90s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.89s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 64.0 / 116  (55.2):  18%|█▊        | 116/651 [1:03:06<12:21:51, 83.20s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.41s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 66.0 / 118  (55.9):  18%|█▊        | 118/651 [1:03:34<7:06:33, 48.02s/it] /Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.71s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 67.0 / 119  (56.3):  18%|█▊        | 119/651 [1:03:51<5:44:08, 38.81s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.82s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 6.85s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 68.0 / 120  (56.7):  18%|█▊        | 120/651 [1:04:24<5:28:00, 37.06s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 10.05s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 71.0 / 124  (57.3):  19%|█▉        | 124/651 [1:05:41<3:20:49, 22.86s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.21s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.23s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.27s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.15s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 71.0 / 125  (56.8):  19%|█▉        | 125/651 [1:07:06<6:04:33, 41.58s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.12s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.30s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 74.0 / 129  (57.4):  20%|█▉        | 129/651 [1:09:25<4:08:19, 28.54s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.49s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 74.0 / 130  (56.9):  20%|█▉        | 130/651 [1:09:58<4:18:39, 29.79s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.84s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 6.40s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 75.0 / 131  (57.3):  20%|██        | 131/651 [1:10:24<4:09:09, 28.75s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.73s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.93s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 75.0 / 132  (56.8):  20%|██        | 132/651 [1:11:03<4:35:15, 31.82s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.69s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.96s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.86s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 76.0 / 134  (56.7):  21%|██        | 134/651 [1:12:17<4:43:30, 32.90s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.03s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 76.0 / 135  (56.3):  21%|██        | 135/651 [1:12:39<4:15:53, 29.75s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.60s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.60s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 77.0 / 136  (56.6):  21%|██        | 136/651 [1:13:23<4:51:26, 33.96s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.17s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.81s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 79.0 / 138  (57.2):  21%|██        | 138/651 [1:14:24<4:22:25, 30.69s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.00s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.52s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.54s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 80.0 / 139  (57.6):  21%|██▏       | 139/651 [1:14:35<3:31:40, 24.81s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.76s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 81.0 / 141  (57.4):  22%|██▏       | 141/651 [1:16:00<4:27:47, 31.51s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.39s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 83.0 / 143  (58.0):  22%|██▏       | 143/651 [1:17:22<4:40:17, 33.10s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.90s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 84.0 / 144  (58.3):  22%|██▏       | 143/651 [1:18:14<4:40:17, 33.10s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.58s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 87.0 / 147  (59.2):  23%|██▎       | 147/651 [1:19:25<4:26:35, 31.74s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.80s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 89.0 / 150  (59.3):  23%|██▎       | 150/651 [1:20:58<3:58:47, 28.60s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 7.75s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 6.01s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 91.0 / 152  (59.9):  23%|██▎       | 152/651 [1:22:46<5:44:50, 41.46s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.14s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 93.0 / 155  (60.0):  24%|██▍       | 155/651 [1:24:23<4:34:42, 33.23s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.37s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 95.0 / 157  (60.5):  24%|██▍       | 156/651 [1:25:03<4:36:18, 33.49s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.13s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 95.0 / 157  (60.5):  24%|██▍       | 157/651 [1:25:05<3:34:19, 26.03s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 8.26s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 97.0 / 159  (61.0):  24%|██▍       | 159/651 [1:26:59<5:32:11, 40.51s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 8.34s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 98.0 / 164  (59.8):  25%|██▌       | 163/651 [1:28:16<2:31:30, 18.63s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t cannot access local variable 'example' where it is not associated with a value\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 98.0 / 164  (59.8):  25%|██▌       | 164/651 [1:28:20<2:27:23, 18.16s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.45s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 98.0 / 166  (59.0):  25%|██▌       | 165/651 [1:28:49<2:38:33, 19.57s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t unmatched ']' (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 98.0 / 166  (59.0):  25%|██▌       | 166/651 [1:28:52<2:10:47, 16.18s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t unmatched ']' (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 98.0 / 167  (58.7):  25%|██▌       | 166/651 [1:29:09<2:10:47, 16.18s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.22s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 98.0 / 167  (58.7):  26%|██▌       | 167/651 [1:29:13<2:23:22, 17.77s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t unmatched ']' (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 98.0 / 168  (58.3):  26%|██▌       | 168/651 [1:29:54<3:19:38, 24.80s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.06s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.02s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.14s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.34s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 101.0 / 174  (58.0):  27%|██▋       | 173/651 [1:31:08<1:50:45, 13.90s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t unmatched ']' (<unknown>, line 1)\n",
      "Error for example in dev set: \t\t unmatched ']' (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 101.0 / 175  (57.7):  27%|██▋       | 175/651 [1:31:12<1:06:41,  8.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t unmatched ']' (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 101.0 / 176  (57.4):  27%|██▋       | 175/651 [1:31:21<1:06:41,  8.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t unmatched ']' (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 101.0 / 178  (56.7):  27%|██▋       | 177/651 [1:31:31<1:07:07,  8.50s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t unmatched ']' (<unknown>, line 1)\n",
      "Error for example in dev set: \t\t unmatched ']' (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 101.0 / 180  (56.1):  28%|██▊       | 180/651 [1:31:41<48:02,  6.12s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t unmatched ']' (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 102.0 / 182  (56.0):  28%|██▊       | 182/651 [1:31:54<46:30,  5.95s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t '[' was never closed (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 102.0 / 184  (55.4):  28%|██▊       | 184/651 [1:31:55<24:27,  3.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t '[' was never closed (<unknown>, line 1)\n",
      "Error for example in dev set: \t\t '[' was never closed (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 104.0 / 187  (55.6):  29%|██▊       | 186/651 [1:32:03<28:05,  3.63s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t '[' was never closed (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 104.0 / 188  (55.3):  29%|██▊       | 187/651 [1:32:09<27:56,  3.61s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t '[' was never closed (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 104.0 / 189  (55.0):  29%|██▉       | 188/651 [1:32:11<29:23,  3.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t '[' was never closed (<unknown>, line 1)\n",
      "Error for example in dev set: \t\t '[' was never closed (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 105.0 / 191  (55.0):  29%|██▉       | 191/651 [1:32:18<24:01,  3.13s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.70s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 105.0 / 192  (54.7):  29%|██▉       | 192/651 [1:32:58<1:47:07, 14.00s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.76s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.31s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.63s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 105.0 / 193  (54.4):  30%|██▉       | 193/651 [1:33:22<2:10:37, 17.11s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.74s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 105.0 / 195  (53.8):  30%|██▉       | 195/651 [1:34:37<3:30:27, 27.69s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.03s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 8.45s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.01s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.57s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 108.0 / 199  (54.3):  31%|███       | 199/651 [1:36:38<2:37:18, 20.88s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.29s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.35s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.26s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.27s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 6.43s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 7.94s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.33s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 111.0 / 204  (54.4):  31%|███▏      | 204/651 [1:40:41<3:53:30, 31.34s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.32s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.40s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 7.00s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 115.0 / 208  (55.3):  32%|███▏      | 208/651 [1:42:48<3:23:11, 27.52s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.31s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.10s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.04s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.18s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 118.0 / 211  (55.9):  32%|███▏      | 211/651 [1:45:16<4:30:54, 36.94s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.05s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 6.67s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 118.0 / 213  (55.4):  33%|███▎      | 213/651 [1:45:58<3:16:59, 26.99s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 6.91s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 119.0 / 214  (55.6):  33%|███▎      | 214/651 [1:46:52<4:14:40, 34.97s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.31s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 122.0 / 218  (56.0):  33%|███▎      | 218/651 [1:48:24<2:37:02, 21.76s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.27s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.10s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.42s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 123.0 / 220  (55.9):  34%|███▍      | 220/651 [1:49:37<3:06:39, 25.98s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.23s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.05s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.09s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 125.0 / 225  (55.6):  34%|███▍      | 224/651 [1:51:01<1:27:31, 12.30s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.29s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 125.0 / 226  (55.3):  35%|███▍      | 226/651 [1:51:05<1:45:03, 14.83s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.30s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 126.0 / 227  (55.5):  35%|███▍      | 227/651 [1:51:18<1:40:40, 14.25s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.92s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 8.06s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.24s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.98s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 126.0 / 228  (55.3):  35%|███▌      | 228/651 [1:52:46<4:16:54, 36.44s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.48s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.02s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 128.0 / 230  (55.7):  35%|███▌      | 230/651 [1:53:22<3:02:21, 25.99s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.44s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.73s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 129.0 / 232  (55.6):  36%|███▌      | 232/651 [1:55:14<4:17:55, 36.94s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.74s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.87s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 132.0 / 238  (55.5):  37%|███▋      | 238/651 [1:56:54<2:13:18, 19.37s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.95s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.38s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 134.0 / 241  (55.6):  37%|███▋      | 241/651 [1:57:30<1:28:27, 12.95s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 8.59s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.19s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 6.06s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.59s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 6.97s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 136.0 / 243  (56.0):  37%|███▋      | 243/651 [2:01:45<7:23:43, 65.25s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.75s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.91s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 138.0 / 246  (56.1):  38%|███▊      | 246/651 [2:02:32<3:24:56, 30.36s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.72s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.69s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 141.0 / 249  (56.6):  38%|███▊      | 249/651 [2:03:14<2:13:37, 19.94s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.32s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.86s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.16s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 142.0 / 250  (56.8):  38%|███▊      | 250/651 [2:05:17<5:41:22, 51.08s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.05s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 142.0 / 252  (56.3):  39%|███▊      | 252/651 [2:05:48<3:30:11, 31.61s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.78s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 143.0 / 253  (56.5):  39%|███▉      | 253/651 [2:06:23<3:36:55, 32.70s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.06s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.25s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 144.0 / 256  (56.2):  39%|███▉      | 256/651 [2:07:55<2:49:39, 25.77s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 6.78s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 148.0 / 260  (56.9):  40%|███▉      | 260/651 [2:08:41<1:13:03, 11.21s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.61s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.40s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 148.0 / 261  (56.7):  40%|████      | 261/651 [2:09:53<3:10:15, 29.27s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.26s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 148.0 / 262  (56.5):  40%|████      | 262/651 [2:10:47<3:58:04, 36.72s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t '[' was never closed (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.31s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 150.0 / 265  (56.6):  41%|████      | 265/651 [2:12:03<3:21:17, 31.29s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.75s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 152.0 / 268  (56.7):  41%|████      | 268/651 [2:12:57<2:07:10, 19.92s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.44s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.96s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 153.0 / 269  (56.9):  41%|████▏     | 269/651 [2:13:18<2:09:21, 20.32s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.98s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 154.0 / 270  (57.0):  41%|████▏     | 270/651 [2:13:41<2:14:40, 21.21s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.92s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 6.62s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 162.0 / 283  (57.2):  43%|████▎     | 283/651 [2:20:34<1:30:19, 14.73s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.98s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 162.0 / 284  (57.0):  44%|████▎     | 284/651 [2:21:00<1:51:59, 18.31s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.31s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 163.0 / 285  (57.2):  44%|████▍     | 285/651 [2:21:28<2:09:45, 21.27s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.89s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 165.0 / 290  (56.9):  45%|████▍     | 290/651 [2:24:17<3:05:57, 30.91s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.55s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 166.0 / 292  (56.8):  45%|████▍     | 292/651 [2:25:03<2:32:49, 25.54s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.22s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.35s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 167.0 / 294  (56.8):  45%|████▌     | 294/651 [2:25:32<1:55:07, 19.35s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.97s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.32s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 167.0 / 295  (56.6):  45%|████▌     | 295/651 [2:26:38<3:17:02, 33.21s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.15s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 169.0 / 300  (56.3):  46%|████▌     | 300/651 [2:27:48<1:38:26, 16.83s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.34s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.53s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 169.0 / 302  (56.0):  46%|████▋     | 302/651 [2:29:09<2:40:11, 27.54s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 28.56s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.84s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 173.0 / 308  (56.2):  47%|████▋     | 308/651 [2:32:00<2:01:13, 21.20s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.08s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 6.84s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 174.0 / 309  (56.3):  47%|████▋     | 309/651 [2:33:16<3:34:50, 37.69s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.26s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.85s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 174.0 / 310  (56.1):  48%|████▊     | 310/651 [2:34:04<3:51:32, 40.74s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.50s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 175.0 / 311  (56.3):  48%|████▊     | 311/651 [2:34:37<3:37:00, 38.30s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.58s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 175.0 / 312  (56.1):  48%|████▊     | 312/651 [2:35:28<3:57:58, 42.12s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 7.80s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 179.0 / 323  (55.4):  50%|████▉     | 323/651 [2:38:47<2:56:56, 32.37s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.03s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 179.0 / 324  (55.2):  50%|████▉     | 324/651 [2:40:13<4:25:08, 48.65s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.66s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 180.0 / 325  (55.4):  50%|████▉     | 325/651 [2:41:02<4:24:58, 48.77s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 7.89s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 184.0 / 330  (55.8):  51%|█████     | 330/651 [2:42:25<1:42:39, 19.19s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.25s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 186.0 / 332  (56.0):  51%|█████     | 332/651 [2:43:21<1:51:59, 21.06s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.99s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 187.0 / 333  (56.2):  51%|█████     | 333/651 [2:45:58<5:27:32, 61.80s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 9.91s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 190.0 / 341  (55.7):  52%|█████▏    | 341/651 [2:49:21<3:16:24, 38.02s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.94s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 194.0 / 348  (55.7):  53%|█████▎    | 348/651 [2:52:34<1:40:41, 19.94s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.84s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 196.0 / 352  (55.7):  54%|█████▍    | 352/651 [2:55:21<3:01:56, 36.51s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.41s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 197.0 / 353  (55.8):  54%|█████▍    | 353/651 [2:55:28<2:21:22, 28.47s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 6.66s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 199.0 / 356  (55.9):  55%|█████▍    | 356/651 [2:56:33<1:46:21, 21.63s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.52s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 200.0 / 357  (56.0):  55%|█████▍    | 357/651 [2:56:57<1:49:21, 22.32s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 7.04s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 201.0 / 358  (56.1):  55%|█████▍    | 358/651 [2:58:22<3:20:40, 41.09s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.89s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 204.0 / 361  (56.5):  55%|█████▌    | 361/651 [3:00:01<2:33:31, 31.76s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.49s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 207.0 / 366  (56.6):  56%|█████▌    | 366/651 [3:01:21<1:11:13, 14.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t source code string cannot contain null bytes\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 208.0 / 370  (56.2):  57%|█████▋    | 370/651 [3:02:02<50:11, 10.72s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t unmatched ']' (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 208.0 / 371  (56.1):  57%|█████▋    | 371/651 [3:02:03<38:03,  8.15s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t unmatched ']' (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 208.0 / 372  (55.9):  57%|█████▋    | 372/651 [3:02:12<39:42,  8.54s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t unmatched ']' (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 208.0 / 373  (55.8):  57%|█████▋    | 373/651 [3:02:16<33:03,  7.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t unmatched ']' (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 208.0 / 375  (55.5):  58%|█████▊    | 375/651 [3:02:31<35:04,  7.63s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t '[' was never closed (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 209.0 / 378  (55.3):  58%|█████▊    | 378/651 [3:02:55<35:54,  7.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t '[' was never closed (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 210.0 / 380  (55.3):  58%|█████▊    | 380/651 [3:03:11<35:25,  7.84s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t unmatched ']' (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 210.0 / 381  (55.1):  59%|█████▊    | 381/651 [3:03:25<44:06,  9.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t '[' was never closed (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 210.0 / 382  (55.0):  59%|█████▊    | 382/651 [3:03:25<31:10,  6.95s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t unmatched ']' (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 6.29s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 210.0 / 383  (54.8):  59%|█████▉    | 383/651 [3:03:37<37:37,  8.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t unmatched ']' (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 210.0 / 384  (54.7):  59%|█████▉    | 384/651 [3:03:37<26:36,  5.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t unmatched ']' (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 210.0 / 385  (54.5):  59%|█████▉    | 385/651 [3:03:38<19:03,  4.30s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t unmatched ']' (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 210.0 / 386  (54.4):  59%|█████▉    | 386/651 [3:03:48<26:46,  6.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t unmatched ']' (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 210.0 / 387  (54.3):  59%|█████▉    | 387/651 [3:03:51<23:13,  5.28s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t unmatched ']' (<unknown>, line 1)\n",
      "Error for example in dev set: \t\t unmatched ']' (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 211.0 / 389  (54.2):  60%|█████▉    | 389/651 [3:04:26<52:49, 12.10s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 6.80s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 212.0 / 392  (54.1):  60%|██████    | 391/651 [3:06:53<2:28:09, 34.19s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.66s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 212.0 / 394  (53.8):  60%|██████    | 393/651 [3:07:26<2:09:37, 30.14s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.77s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 216.0 / 398  (54.3):  61%|██████    | 398/651 [3:09:06<1:32:19, 21.90s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.10s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 217.0 / 399  (54.4):  61%|██████▏   | 399/651 [3:10:01<2:14:09, 31.94s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.88s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.21s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 217.0 / 400  (54.2):  61%|██████▏   | 400/651 [3:10:36<2:17:44, 32.93s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.14s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 218.0 / 402  (54.2):  62%|██████▏   | 402/651 [3:11:25<1:50:42, 26.67s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.62s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.27s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 219.0 / 404  (54.2):  62%|██████▏   | 404/651 [3:12:36<1:57:46, 28.61s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.12s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 221.0 / 406  (54.4):  62%|██████▏   | 406/651 [3:13:33<1:50:41, 27.11s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.74s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.76s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error during API call: Error code: 400 - {'error': {'message': \"'$.input' is invalid. Please check the API reference: https://platform.openai.com/docs/api-reference.\", 'type': 'invalid_request_error', 'param': None, 'code': None}}\n",
      "Error for example in dev set: \t\t shapes (0,) and (1536,) not aligned: 0 (dim 0) != 1536 (dim 0)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 224.0 / 414  (54.1):  64%|██████▎   | 414/651 [3:16:56<1:49:22, 27.69s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 6.57s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.40s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 224.0 / 415  (54.0):  64%|██████▎   | 415/651 [3:18:26<3:02:26, 46.38s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.24s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.79s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 225.0 / 416  (54.1):  64%|██████▍   | 416/651 [3:19:10<2:59:34, 45.85s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.19s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 8.38s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 227.0 / 421  (53.9):  65%|██████▍   | 421/651 [3:21:13<1:35:42, 24.97s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.18s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.89s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 227.0 / 422  (53.8):  65%|██████▍   | 422/651 [3:22:02<2:02:47, 32.17s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.51s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 228.0 / 425  (53.6):  65%|██████▌   | 425/651 [3:23:34<1:43:47, 27.56s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.33s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 229.0 / 428  (53.5):  66%|██████▌   | 428/651 [3:25:28<2:12:11, 35.57s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 7.91s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 230.0 / 430  (53.5):  66%|██████▌   | 430/651 [3:27:27<2:44:24, 44.64s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.23s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.20s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 232.0 / 432  (53.7):  66%|██████▋   | 432/651 [3:28:09<1:57:13, 32.12s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 8.17s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 28.30s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 234.0 / 434  (53.9):  67%|██████▋   | 434/651 [3:30:16<2:43:46, 45.29s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.95s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 235.0 / 436  (53.9):  67%|██████▋   | 436/651 [3:30:41<1:35:20, 26.61s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.39s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 236.0 / 438  (53.9):  67%|██████▋   | 438/651 [3:31:20<1:21:11, 22.87s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.78s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 237.0 / 439  (54.0):  67%|██████▋   | 439/651 [3:33:05<2:48:30, 47.69s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 10.56s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 238.0 / 440  (54.1):  68%|██████▊   | 440/651 [3:34:07<3:02:18, 51.84s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 7.78s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.07s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 244.0 / 449  (54.3):  69%|██████▉   | 449/651 [3:37:10<41:26, 12.31s/it]  /Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/PyPDF2/_cmap.py:142: PdfReadWarning: Advanced encoding /SymbolSetEncoding not implemented yet\n",
      "  warnings.warn(\n",
      "Average Metric: 244.0 / 450  (54.2):  69%|██████▉   | 450/651 [3:38:06<1:25:02, 25.39s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.42s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 245.0 / 453  (54.1):  70%|██████▉   | 453/651 [3:39:04<1:00:01, 18.19s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 6.87s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.86s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 247.0 / 458  (53.9):  70%|███████   | 458/651 [3:40:04<37:59, 11.81s/it]  /Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.92s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 249.0 / 461  (54.0):  71%|███████   | 460/651 [3:40:54<53:29, 16.80s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.41s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 252.0 / 464  (54.3):  71%|███████▏  | 464/651 [3:41:21<30:39,  9.84s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.67s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.48s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.56s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 254.0 / 466  (54.5):  72%|███████▏  | 466/651 [3:42:30<59:31, 19.30s/it]  /Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.57s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.81s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 6.46s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 255.0 / 471  (54.1):  72%|███████▏  | 471/651 [3:45:22<1:05:50, 21.95s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 6.36s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 256.0 / 475  (53.9):  73%|███████▎  | 475/651 [3:45:50<28:47,  9.82s/it]  /Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.18s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 6.12s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 257.0 / 477  (53.9):  73%|███████▎  | 477/651 [3:46:55<1:00:16, 20.78s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 8.10s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 9.57s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 258.0 / 478  (54.0):  73%|███████▎  | 478/651 [3:49:52<3:14:40, 67.52s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 8.40s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.25s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.91s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 263.0 / 486  (54.1):  75%|███████▍  | 486/651 [3:52:30<1:03:58, 23.26s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.95s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 263.0 / 489  (53.8):  75%|███████▌  | 489/651 [3:54:15<1:16:17, 28.26s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 6.37s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.36s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 268.0 / 498  (53.8):  76%|███████▋  | 498/651 [3:57:49<46:45, 18.34s/it]  /Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.34s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 269.0 / 499  (53.9):  77%|███████▋  | 499/651 [3:58:57<1:23:18, 32.88s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.59s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 269.0 / 500  (53.8):  77%|███████▋  | 500/651 [3:59:11<1:08:52, 27.37s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.84s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.01s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 270.0 / 502  (53.8):  77%|███████▋  | 502/651 [4:01:20<1:46:05, 42.72s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.82s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 271.0 / 508  (53.3):  78%|███████▊  | 507/651 [4:02:30<34:56, 14.56s/it]  /Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.33s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 271.0 / 509  (53.2):  78%|███████▊  | 509/651 [4:02:38<22:21,  9.45s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.88s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 271.0 / 511  (53.0):  78%|███████▊  | 511/651 [4:03:49<51:32, 22.09s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.61s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 272.0 / 513  (53.0):  79%|███████▉  | 513/651 [4:04:29<48:31, 21.10s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.39s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 272.0 / 515  (52.8):  79%|███████▉  | 514/651 [4:05:42<1:05:36, 28.73s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t '[' was never closed (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 272.0 / 515  (52.8):  79%|███████▉  | 515/651 [4:05:45<1:05:45, 29.01s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.32s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 272.0 / 516  (52.7):  79%|███████▉  | 516/651 [4:06:11<1:03:14, 28.11s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t '[' was never closed (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 272.0 / 518  (52.5):  79%|███████▉  | 517/651 [4:06:48<1:00:38, 27.15s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t '[' was never closed (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 275.0 / 522  (52.7):  80%|████████  | 521/651 [4:07:13<26:08, 12.07s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t '[' was never closed (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 275.0 / 522  (52.7):  80%|████████  | 522/651 [4:07:13<18:45,  8.72s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 0.85s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 277.0 / 525  (52.8):  80%|████████  | 524/651 [4:07:38<19:09,  9.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t '[' was never closed (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 277.0 / 526  (52.7):  81%|████████  | 526/651 [4:07:41<12:05,  5.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t '[' was never closed (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 277.0 / 527  (52.6):  81%|████████  | 527/651 [4:08:42<46:41, 22.59s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t '[' was never closed (<unknown>, line 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 277.0 / 528  (52.5):  81%|████████  | 528/651 [4:09:26<59:29, 29.02s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.93s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 277.0 / 529  (52.4):  81%|████████  | 528/651 [4:11:34<59:29, 29.02s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.99s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 280.0 / 535  (52.3):  82%|████████▏ | 535/651 [4:14:25<45:13, 23.39s/it]  /Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.78s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 282.0 / 538  (52.4):  83%|████████▎ | 538/651 [4:16:08<54:57, 29.18s/it]  /Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 8.25s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.65s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.50s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 289.0 / 552  (52.4):  85%|████████▍ | 552/651 [4:21:20<25:33, 15.49s/it]  /Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 8.44s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 289.0 / 553  (52.3):  85%|████████▍ | 553/651 [4:22:09<41:44, 25.55s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 7.96s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.09s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 290.0 / 554  (52.3):  85%|████████▌ | 554/651 [4:22:44<45:45, 28.30s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 6.22s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 290.0 / 556  (52.2):  85%|████████▌ | 556/651 [4:23:46<45:26, 28.70s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.61s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 291.0 / 560  (52.0):  86%|████████▌ | 559/651 [4:24:31<28:44, 18.75s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t PyCryptodome is required for AES algorithm\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 291.0 / 561  (51.9):  86%|████████▌ | 560/651 [4:25:08<24:49, 16.36s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.96s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 292.0 / 562  (52.0):  86%|████████▌ | 561/651 [4:25:20<30:50, 20.57s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.52s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 295.0 / 568  (51.9):  87%|████████▋ | 568/651 [4:27:07<19:02, 13.77s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.85s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 295.0 / 569  (51.8):  87%|████████▋ | 569/651 [4:28:09<38:25, 28.12s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.55s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 7.70s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.53s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.06s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 298.0 / 572  (52.1):  88%|████████▊ | 572/651 [4:30:56<51:34, 39.17s/it]  /Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.38s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 303.0 / 579  (52.3):  89%|████████▉ | 579/651 [4:33:42<28:21, 23.63s/it]  /Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.86s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.67s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.93s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 306.0 / 586  (52.2):  90%|█████████ | 586/651 [4:35:49<11:19, 10.46s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.98s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.75s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 306.0 / 587  (52.1):  90%|█████████ | 587/651 [4:36:58<29:00, 27.19s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.54s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 6.34s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 307.0 / 589  (52.1):  90%|█████████ | 589/651 [4:38:04<28:25, 27.51s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 7.52s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 308.0 / 592  (52.0):  91%|█████████ | 592/651 [4:39:36<24:31, 24.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error for example in dev set: \t\t negative seek value -1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.07s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.01s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 312.0 / 596  (52.3):  92%|█████████▏| 596/651 [4:41:10<17:49, 19.44s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.72s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 312.0 / 597  (52.3):  92%|█████████▏| 597/651 [4:41:27<17:00, 18.90s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 10.23s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 314.0 / 601  (52.2):  92%|█████████▏| 601/651 [4:43:04<13:56, 16.73s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 3.66s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 318.0 / 609  (52.2):  93%|█████████▎| 608/651 [4:45:32<10:26, 14.57s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.37s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 318.0 / 609  (52.2):  94%|█████████▎| 609/651 [4:45:35<11:20, 16.20s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 6.68s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 319.0 / 610  (52.3):  94%|█████████▎| 610/651 [4:45:59<12:47, 18.71s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 2.51s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 320.0 / 613  (52.2):  94%|█████████▍| 613/651 [4:46:36<09:27, 14.92s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 6.88s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 324.0 / 619  (52.3):  95%|█████████▌| 619/651 [4:51:16<18:06, 33.97s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.63s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 326.0 / 621  (52.5):  95%|█████████▌| 621/651 [4:52:24<17:42, 35.43s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.58s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 327.0 / 622  (52.6):  96%|█████████▌| 622/651 [4:53:51<24:12, 50.08s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 4.70s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 334.0 / 631  (52.9):  97%|█████████▋| 631/651 [4:59:19<15:09, 45.48s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.68s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 335.0 / 633  (52.9):  97%|█████████▋| 633/651 [5:00:02<09:28, 31.57s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 5.13s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 340.0 / 640  (53.1):  98%|█████████▊| 640/651 [5:03:22<04:47, 26.13s/it]/Users/codyaldaz/repositories/dspy_scirepeval/venv/lib/python3.12/site-packages/dsp/modules/gpt3.py:264: UserWarning: Persisting input arguments took 1.37s to run.If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function.\n",
      "  return v1_cached_gpt3_turbo_request_v2(**kwargs)\n",
      "Average Metric: 344.0 / 651  (52.8): 100%|██████████| 651/651 [5:13:48<00:00, 28.92s/it] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average Metric: 344.0 / 651  (52.8%)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate = Evaluate(devset=trainset, metric=metric, num_threads=8, display_progress=True, display_table=0, max_errors=100, return_outputs=True)\n",
    "outputs = evaluate(pipeline_chunking_retrieval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "Predict if the two chunks are related by a citation. Consider all possible ways in which a citation could occur, such as direct quotes, paraphrasing, or referring to the same ideas or data. Don't be afraid to predict that the chunks are related by a citation. If you're not sure, it's better to predict that they are related.\n",
      "\n",
      "---\n",
      "\n",
      "Follow the following format.\n",
      "\n",
      "Query Chunk: Query chunk to compare to the candidate chunk.\n",
      "\n",
      "Candidate Chunk: Candidate chunk to compare to the query chunk.\n",
      "\n",
      "Example: A good example to learn from.\n",
      "\n",
      "Reasoning: Let's think step by step in order to ${produce the answer}. We ...\n",
      "\n",
      "Answer: either True or False\n",
      "\n",
      "---\n",
      "\n",
      "Query Chunk:  m nmmm vvvv  321CNN Extracted descriptor vector sequence 1st-person activity video … … Time series representation Per-frame feature representation … Temporal pooling Final representation sum pooling ‘histogram of time series gradients’ pooling … … … … … n*m dimensional data (e.g., n = 4096 features, m = 1000 frames) n time series (e.g., 4096) k temporal filters (e.g., 15) n*k-D vector (e.g., 61440) … max pooling Figure 2. Overall representation framework of our pooled time series (PoT). PoT feature representation of the video: x= [xop1 1[ts 1;te 1]; xop2 1[ts 1;te 1];\u0001\u0001\u0001; xopr n[ts k;te k]] (1) wherexopj ispeciﬁes that it is applying the jth pooling op- erator to the ith time series fi(t). Our PoT representation takes advantage of four different types of pooling operators including two newly introduced temporal pooling operators, which we discuss more in Subsection 2.2. Our framework of (i) extracting per-frame descriptors, (ii) interpreting them as a set of time series\n",
      "\n",
      "Candidate Chunk: might constitute a good starting point forfurtherdevelopment. Accordingly,severalauthorsattemptedtoextenditbyaddressingitsmain perceived limitation—its inability to deal with attribute dependences. 106 P. DOMINGOS AND M. PAZZANI Langley and Sage (1994) argued that, when two attributes are correlated, it might be better to delete one attribute than to assume the two are conditionally independent. Theyfoundthatanalgorithmforfeaturesubsetselection(forwardsequentialselection)improvedaccuracyonsomedatasets,buthadlittleornoeffectinothers. Inarelatedapproach,Kubat,Flotzinger,andPfurtscheller(1993)foundthatusingadecision-treelearnertoselectfeaturesforuseintheBayesianclassiﬁergavegoodresultsinthedomainofEEGsignalclassiﬁcation. Kononenko(1991)proposedsuccessivelyjoiningdependentattributevalues,usingasta- tisticaltesttojudgewhethertwoattributevaluesaresigniﬁcantlydependent. Experimentalresults with this method were not encouraging. On two domains, the modiﬁed Bayesianclassiﬁer had the same accuracy as the simple Ba\n",
      "\n",
      "Example: Query Chunk: stacked Point- Net (PN) layers. The ﬁrst layer consumes minput pointsrepresented as an m\u00023matrixPwhere each row is the 3D coordinate of a point pi= (x;y;z ). A shared multi- layer perceptron (MLP) consisting of two linear layers with ReLU activation is used to transform each piinto a point feature vector fi. This gives us a feature matrix Fwhose rows are the learned point features fi. Then, a point-wise maxpooling is performed on Fto obtain ak-dimensional global feature g, where gj= maxi=1;:::;mfFijgforj= 1;:::;k . The second PN layer takes Fandgas input. It ﬁrst concatenates gto each fito obtain an augmented point feature matrix eFwhose rows are the concatenated feature vectors [fig]. Then,eFis passed through another shared MLP and point-wise max pooling similar to the ones in the ﬁrst layer, which gives the ﬁnal feature vector v. 4.2. Multistage Point Generation The decoder is responsible for generating the output point cloud from the feature vector v. Our proposed de- coder combines the advantages of Candidate Chunk: ple kernels (MK-MMD) [27] for adapting deep representations. Wasserstein Distance Guided Representation Learning (WDGRL) [28] minimizes the dis- tribution discrepancy by employing Wasserstein Distance in neural networks. However, these approaches have a strong assumption that there is a uniﬁed transformation to map the source domain and target domain into a common space in which the feature representations are domain invariant. (2) Extracting intermediate features to minimize distribution discrepancy by projecting data onto subspaces or transforming a source subspace into a new subspace, which is related to the target subspace. Geodesic Flow Kernel (GFK) [8] considersthe intermediate information in geodesic curve between the source domain and target domain on a Grassmann manifold. Subspace alignment (SA) [9] maps a source PCA subspace into a new subspace which is well-aligned with the target subspace. Correlation Alignment (CORAL) [10] matches the covariance matrix of the source subspace and target subspa Answer: False\n",
      "\n",
      "Reasoning: Let's think step by step in order to False The query chunk discusses the representation framework of pooled time series in video analysis, while the candidate chunk talks about different approaches for adapting deep representations and minimizing distribution discrepancy in neural networks. There is no direct relationship between the two chunks in terms of content or context, so they are not related by a citation.\n",
      "\n",
      "Answer:\u001b[32m False\u001b[0m\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Predict if the two chunks are related by a citation. Consider all possible ways in which a citation could occur, such as direct quotes, paraphrasing, or referring to the same ideas or data. Don't be afraid to predict that the chunks are related by a citation. If you're not sure, it's better to predict that they are related.\n",
      "\n",
      "---\n",
      "\n",
      "Follow the following format.\n",
      "\n",
      "Query Chunk: Query chunk to compare to the candidate chunk.\n",
      "\n",
      "Candidate Chunk: Candidate chunk to compare to the query chunk.\n",
      "\n",
      "Example: A good example to learn from.\n",
      "\n",
      "Reasoning: Let's think step by step in order to ${produce the answer}. We ...\n",
      "\n",
      "Answer: either True or False\n",
      "\n",
      "---\n",
      "\n",
      "Query Chunk: , and (iii) apply- ing various types of time series pooling and concatenating them provides the following three important abilities: First, (1) it preserves detailed dynamics displayed in each descriptor element as a time series, and allows the rep- resentation to capture both long-term motion and short-term information with multiple temporal ﬁlters. That is, depend- ing on the nature of the time series, our representation is able to capture subtle short-term motion by pooling from a ﬁlter with a small time interval as well as long-term motion by performing pooling with a large time interval. Such ﬂex- ibility is in contrast to previous bag-of-visual-words repre- sentation for global motion descriptors (e.g., the one used in [18]) that abstracts all descriptor values in one frame (or a subsequence of few frames) into a single discretized ‘vi- sual word’. In addition, (2) our representation explicitly im- poses temporal structure of the activity by decomposing the entire time interval to multiple subinterv\n",
      "\n",
      "Candidate Chunk: might constitute a good starting point forfurtherdevelopment. Accordingly,severalauthorsattemptedtoextenditbyaddressingitsmain perceived limitation—its inability to deal with attribute dependences. 106 P. DOMINGOS AND M. PAZZANI Langley and Sage (1994) argued that, when two attributes are correlated, it might be better to delete one attribute than to assume the two are conditionally independent. Theyfoundthatanalgorithmforfeaturesubsetselection(forwardsequentialselection)improvedaccuracyonsomedatasets,buthadlittleornoeffectinothers. Inarelatedapproach,Kubat,Flotzinger,andPfurtscheller(1993)foundthatusingadecision-treelearnertoselectfeaturesforuseintheBayesianclassiﬁergavegoodresultsinthedomainofEEGsignalclassiﬁcation. Kononenko(1991)proposedsuccessivelyjoiningdependentattributevalues,usingasta- tisticaltesttojudgewhethertwoattributevaluesaresigniﬁcantlydependent. Experimentalresults with this method were not encouraging. On two domains, the modiﬁed Bayesianclassiﬁer had the same accuracy as the simple Ba\n",
      "\n",
      "Example: Query Chunk: retains its class identity no mat- ter how it is depicted: a dog is a dog whether photographed, painted in oils, or drawn with a stick in the sand. Cross-depiction is a practical problem too: an example is an image search. The world contains images in all sorts of depictions. Any recognition solution that does not generalise across these depictions is of limited power. Yet most current computer vision methods tacitly assume a photographic input, either by design or training. Any model premised on a single depictive style e.g. photos will lack su\u000ecient descriptive power for cross-depiction recognition. Therefore, an image search using methods will limit its results to photos and photo-like depictions. In our paper, we talk about natural images (photos) and non-natural im- ages (artwork) as a linguistic convenience. We would argue that this is a false dichotomy: the universe of all images includes images in all possible depictive styles, and there is no particular reason to privilege any one style. Neverth Candidate Chunk: are assumed to be conditionally independent. Even in spite of assumption of the features dependencies, in practice, they provide satisfying results, they are very easy to implement and fast to evaluate, where they just need a small number of training data to estimate their parameters. However, the main disadvantage is where the Bayesian classiﬁers make a very strong assumption on the shape of data distribution. In addition, they can not learn interactions between the features, and suﬀer from zero conditional probability problem (division by zero) [18], where one simple solution would be to add some implicit examples. Furthermore, computing the probabilities for continuous features is not possible by the traditional method of frequency counts. Nevertheless, some studies have found that, with an appropriate pre-processing, Bayesian classiﬁers can be comparable in performance with other classiﬁcation algorithms [10, 19, 20, 21]. Neural networks are modeled based on the same analogy to the human brain workin Answer: False\n",
      "\n",
      "Reasoning: Let's think step by step in order to\u001b[32m compare the two chunks. The query chunk discusses the application of time series pooling and concatenation to capture detailed dynamics in motion descriptors, while the candidate chunk talks about the limitations and advantages of Bayesian classifiers and neural networks in handling feature dependencies. There is no direct connection between the content of the two chunks, as they discuss different topics related to machine learning techniques. Therefore, they are not related by a citation.\n",
      "\n",
      "Answer: False\u001b[0m\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Predict if the two chunks are related by a citation. Consider all possible ways in which a citation could occur, such as direct quotes, paraphrasing, or referring to the same ideas or data. Don't be afraid to predict that the chunks are related by a citation. If you're not sure, it's better to predict that they are related.\n",
      "\n",
      "---\n",
      "\n",
      "Follow the following format.\n",
      "\n",
      "Query Chunk: Query chunk to compare to the candidate chunk.\n",
      "\n",
      "Candidate Chunk: Candidate chunk to compare to the query chunk.\n",
      "\n",
      "Example: A good example to learn from.\n",
      "\n",
      "Reasoning: Let's think step by step in order to ${produce the answer}. We ...\n",
      "\n",
      "Answer: either True or False\n",
      "\n",
      "---\n",
      "\n",
      "Query Chunk: als, which is very important for representing high-level activities. Finally, (3) it allows the system to take advantage of multiple types of pooling operators so that the representation captures differ- ent aspects of the data. As a result of our framework, each video is represented with one single vector having a tractable dimensionality.Activity recognition is performed by training/testing stan- dard classiﬁers (e.g., SVM classiﬁers) with these vectors. Our representation is able to cope with any type of genera- tive and discriminative classiﬁers in principle, and we show its superiority over others in Section 3.2. 2.1. Handling high-dimensional feature descriptors The proposed representation framework is very gen- eral in the aspect that it is able to cope with any types of per-frame image/motion descriptors such as histogram of oriented gradients (HOG) or histogram of optical ﬂows (HOF). Furthermore, it is particularly designed to han- dle high-dimensional per-frame image descriptors: image- based de\n",
      "\n",
      "Candidate Chunk: might constitute a good starting point forfurtherdevelopment. Accordingly,severalauthorsattemptedtoextenditbyaddressingitsmain perceived limitation—its inability to deal with attribute dependences. 106 P. DOMINGOS AND M. PAZZANI Langley and Sage (1994) argued that, when two attributes are correlated, it might be better to delete one attribute than to assume the two are conditionally independent. Theyfoundthatanalgorithmforfeaturesubsetselection(forwardsequentialselection)improvedaccuracyonsomedatasets,buthadlittleornoeffectinothers. Inarelatedapproach,Kubat,Flotzinger,andPfurtscheller(1993)foundthatusingadecision-treelearnertoselectfeaturesforuseintheBayesianclassiﬁergavegoodresultsinthedomainofEEGsignalclassiﬁcation. Kononenko(1991)proposedsuccessivelyjoiningdependentattributevalues,usingasta- tisticaltesttojudgewhethertwoattributevaluesaresigniﬁcantlydependent. Experimentalresults with this method were not encouraging. On two domains, the modiﬁed Bayesianclassiﬁer had the same accuracy as the simple Ba\n",
      "\n",
      "Example: Query Chunk: retains its class identity no mat- ter how it is depicted: a dog is a dog whether photographed, painted in oils, or drawn with a stick in the sand. Cross-depiction is a practical problem too: an example is an image search. The world contains images in all sorts of depictions. Any recognition solution that does not generalise across these depictions is of limited power. Yet most current computer vision methods tacitly assume a photographic input, either by design or training. Any model premised on a single depictive style e.g. photos will lack su\u000ecient descriptive power for cross-depiction recognition. Therefore, an image search using methods will limit its results to photos and photo-like depictions. In our paper, we talk about natural images (photos) and non-natural im- ages (artwork) as a linguistic convenience. We would argue that this is a false dichotomy: the universe of all images includes images in all possible depictive styles, and there is no particular reason to privilege any one style. Neverth Candidate Chunk: are assumed to be conditionally independent. Even in spite of assumption of the features dependencies, in practice, they provide satisfying results, they are very easy to implement and fast to evaluate, where they just need a small number of training data to estimate their parameters. However, the main disadvantage is where the Bayesian classiﬁers make a very strong assumption on the shape of data distribution. In addition, they can not learn interactions between the features, and suﬀer from zero conditional probability problem (division by zero) [18], where one simple solution would be to add some implicit examples. Furthermore, computing the probabilities for continuous features is not possible by the traditional method of frequency counts. Nevertheless, some studies have found that, with an appropriate pre-processing, Bayesian classiﬁers can be comparable in performance with other classiﬁcation algorithms [10, 19, 20, 21]. Neural networks are modeled based on the same analogy to the human brain workin Answer: False\n",
      "\n",
      "Reasoning: Let's think step by step in order to\u001b[32m produce the answer. We can see that the query chunk discusses a framework for activity recognition and representation of high-level activities using pooling operators, while the candidate chunk talks about Bayesian classifiers and their assumptions on feature dependencies. There is no direct connection or overlap in content between the two chunks, so they are not related by a citation.\n",
      "\n",
      "Answer: False\u001b[0m\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Predict if the two chunks are related by a citation. Consider all possible ways in which a citation could occur, such as direct quotes, paraphrasing, or referring to the same ideas or data. Don't be afraid to predict that the chunks are related by a citation. If you're not sure, it's better to predict that they are related.\n",
      "\n",
      "---\n",
      "\n",
      "Follow the following format.\n",
      "\n",
      "Query Chunk: Query chunk to compare to the candidate chunk.\n",
      "\n",
      "Candidate Chunk: Candidate chunk to compare to the query chunk.\n",
      "\n",
      "Example: A good example to learn from.\n",
      "\n",
      "Reasoning: Let's think step by step in order to ${produce the answer}. We ...\n",
      "\n",
      "Answer: either True or False\n",
      "\n",
      "---\n",
      "\n",
      "Query Chunk: ep learning features which are also called convolu- tional neural network (CNN) features [7, 19]. These deep learning image features are obtained by snatching interme- diate outputs from internal convolutional layers of a CNN, pre-trained on image datasets. They can also be viewed as cascades of automatically learned image ﬁlters. These im- age descriptors are trained from large scale image datasets and have obtained highly successful results on image clas- siﬁcation/detection [4] as well as video classiﬁcation [8], performing superior to state-of-the-art hand-designed im- age descriptors such as HOG even without re-training the networks. Our motivation was to design a general representation that best takes advantage of such high-dimensional descrip- tors and conﬁrm that these CNN features are able to in- crease ﬁrst-person activity recognition performance signif- icantly together with other features. Each element of a CNN feature vector abstracts particular local/global appear- ance for a single frame, a\n",
      "\n",
      "Candidate Chunk: s, and outputs a classiﬁer. The classiﬁer takes an unlabeledexample and assigns it to a class. Many classiﬁers can be viewed as computing a set ofdiscriminantfunctions oftheexample,oneforeachclass,andassigningtheexampletothe classwhosefunctionismaximum(Duda&Hart,1973). If Eistheexample,and f i(E)is the discriminant function corresponding to the ith class, the chosen class Ckis the one for which1 fk(E)>fi(E)8i6=k: (1) Suppose an example is a vector of aattributes, as is typically the case in classiﬁcation ap- plications. Let vjkbethevalueofattribute Ajintheexample, P(X)denotetheprobability ofX,andP(YjX)denotetheconditionalprobabilityof YgivenX. Thenonepossibleset of discriminant functions is fi(E)=P(Ci)aY j=1P(Aj=vjkjCi): (2) 104 P. DOMINGOS AND M. PAZZANI The classiﬁer obtained by using this set of discriminant functions, and estimating the relevant probabilities from the training set, is often called the naive Bayesian classiﬁer . Thisisbecause,ifthe“naive”assumptionismadethattheattributesareindependentg\n",
      "\n",
      "Example: Query Chunk: retains its class identity no mat- ter how it is depicted: a dog is a dog whether photographed, painted in oils, or drawn with a stick in the sand. Cross-depiction is a practical problem too: an example is an image search. The world contains images in all sorts of depictions. Any recognition solution that does not generalise across these depictions is of limited power. Yet most current computer vision methods tacitly assume a photographic input, either by design or training. Any model premised on a single depictive style e.g. photos will lack su\u000ecient descriptive power for cross-depiction recognition. Therefore, an image search using methods will limit its results to photos and photo-like depictions. In our paper, we talk about natural images (photos) and non-natural im- ages (artwork) as a linguistic convenience. We would argue that this is a false dichotomy: the universe of all images includes images in all possible depictive styles, and there is no particular reason to privilege any one style. Neverth Candidate Chunk: are assumed to be conditionally independent. Even in spite of assumption of the features dependencies, in practice, they provide satisfying results, they are very easy to implement and fast to evaluate, where they just need a small number of training data to estimate their parameters. However, the main disadvantage is where the Bayesian classiﬁers make a very strong assumption on the shape of data distribution. In addition, they can not learn interactions between the features, and suﬀer from zero conditional probability problem (division by zero) [18], where one simple solution would be to add some implicit examples. Furthermore, computing the probabilities for continuous features is not possible by the traditional method of frequency counts. Nevertheless, some studies have found that, with an appropriate pre-processing, Bayesian classiﬁers can be comparable in performance with other classiﬁcation algorithms [10, 19, 20, 21]. Neural networks are modeled based on the same analogy to the human brain workin Answer: False\n",
      "\n",
      "Reasoning: Let's think step by step in order to\u001b[32m compare the two chunks. The first chunk discusses deep learning image features obtained from convolutional neural networks, specifically focusing on their success in image classification and detection tasks. On the other hand, the second chunk delves into the concept of Bayesian classifiers, their assumptions, advantages, and disadvantages. While both chunks discuss machine learning techniques, they focus on different aspects - one on deep learning image features and the other on Bayesian classifiers. There is no direct indication or reference in either chunk to suggest that they are related by a citation. Therefore, the chunks are not related by a citation. \n",
      "\n",
      "Answer: False\u001b[0m\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Predict if the two chunks are related by a citation. Consider all possible ways in which a citation could occur, such as direct quotes, paraphrasing, or referring to the same ideas or data. Don't be afraid to predict that the chunks are related by a citation. If you're not sure, it's better to predict that they are related.\n",
      "\n",
      "---\n",
      "\n",
      "Follow the following format.\n",
      "\n",
      "Query Chunk: Query chunk to compare to the candidate chunk.\n",
      "\n",
      "Candidate Chunk: Candidate chunk to compare to the query chunk.\n",
      "\n",
      "Example: A good example to learn from.\n",
      "\n",
      "Reasoning: Let's think step by step in order to ${produce the answer}. We ...\n",
      "\n",
      "Answer: either True or False\n",
      "\n",
      "---\n",
      "\n",
      "Query Chunk: nd (by extension) its time se- ries models how this local/global appearance is changing over time. As a human in the scene moves (e.g., changes his/her posture) and the camera changes its viewpoint be- cause of egomotion, certain CNN feature values will be- come activated/deactivated and our idea is to keep track of such changes to represent the activity video. In our experi- ments, we explicitly conﬁrm this while comparing our rep- resentation with the conventional representations. When us- ing CNN features as our base per-frame descriptors, we get a 4096-dimensional feature vector (i.e., n=4096) for each image frame by obtaining outputs of the last convolutional layer (e.g., stage 7 in [19]). 2.2. Temporal pooling operators Our PoT representation is constructed by applying multi- ple types of temporal pooling operators over each temporal ﬁlter (i.e., time interval). In this paper, we take advantage of four different types of pooling operators: conventional max pooling and sum pooling, and two types of o\n",
      "\n",
      "Candidate Chunk: PLE BAYESIAN CLASSIFIER 105 Theremainderofthearticleelaboratesontheseideas. Section2reviewspreviousempir- icalresultsontheBayesianclassiﬁerinthemachinelearningliterature,andrecentattemptsto extend it. Section 3 describes an empirical study showing that the Bayesian classiﬁeroutperformsseveralmoresophisticatedapproachesonalargenumberofdatasets,andthatthisisnotduetotheabsenceofattributedependencesinthosedatasets. Section4presentsasimpleexamplethatillustratessomeofthekeypointstobemadesubsequently. Section5derivesnecessaryandsufﬁcientconditionsforthelocaloptimalityoftheBayesianclassiﬁer(i.e.,itsoptimalityforanygivenexample),andcomputeshowoftentheseconditionswillbesatisﬁed. Section 6 generalizes the previous results to a necessary and sufﬁcient conditionfor the Bayesian classiﬁer’s global optimality (i.e., its optimality for any given data set).It also shows that the Bayesian classiﬁer has some fundamental limitations, but is optimalfor learning conjunctions and disjunctions. Section 7 formulates some hypothes\n",
      "\n",
      "Example: Query Chunk: retains its class identity no mat- ter how it is depicted: a dog is a dog whether photographed, painted in oils, or drawn with a stick in the sand. Cross-depiction is a practical problem too: an example is an image search. The world contains images in all sorts of depictions. Any recognition solution that does not generalise across these depictions is of limited power. Yet most current computer vision methods tacitly assume a photographic input, either by design or training. Any model premised on a single depictive style e.g. photos will lack su\u000ecient descriptive power for cross-depiction recognition. Therefore, an image search using methods will limit its results to photos and photo-like depictions. In our paper, we talk about natural images (photos) and non-natural im- ages (artwork) as a linguistic convenience. We would argue that this is a false dichotomy: the universe of all images includes images in all possible depictive styles, and there is no particular reason to privilege any one style. Neverth Candidate Chunk: are assumed to be conditionally independent. Even in spite of assumption of the features dependencies, in practice, they provide satisfying results, they are very easy to implement and fast to evaluate, where they just need a small number of training data to estimate their parameters. However, the main disadvantage is where the Bayesian classiﬁers make a very strong assumption on the shape of data distribution. In addition, they can not learn interactions between the features, and suﬀer from zero conditional probability problem (division by zero) [18], where one simple solution would be to add some implicit examples. Furthermore, computing the probabilities for continuous features is not possible by the traditional method of frequency counts. Nevertheless, some studies have found that, with an appropriate pre-processing, Bayesian classiﬁers can be comparable in performance with other classiﬁcation algorithms [10, 19, 20, 21]. Neural networks are modeled based on the same analogy to the human brain workin Answer: False\n",
      "\n",
      "Reasoning: Let's think step by step in order to\u001b[32m produce the answer. The query chunk discusses the use of CNN features and temporal pooling operators in representing activity videos, while the candidate chunk talks about the Bayesian classifier, its assumptions, advantages, and disadvantages. There is no direct connection between the two chunks in terms of content or ideas. They discuss different concepts and methodologies in machine learning. Therefore, they are not related by a citation.\n",
      "\n",
      "Answer: False\u001b[0m\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "llm.inspect_history(n=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "651\n"
     ]
    }
   ],
   "source": [
    "all_predictions = []\n",
    "for x in outputs[1]:\n",
    "    if type(x[1])==dspy.Prediction:\n",
    "        all_predictions.append(x[1].resolved)\n",
    "    else:\n",
    "        all_predictions.append(np.nan)\n",
    "    \n",
    "\n",
    "all_labels = [x[0].cites for x in outputs[1]]\n",
    "print(len(all_predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./predictions_one_shot_prompt_1000.txt', 'w') as f:\n",
    "    for pred in all_predictions:\n",
    "        f.write(str(pred) + '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "651\n"
     ]
    }
   ],
   "source": [
    "print(len(all_predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.53\n",
      "Recall:  0.57\n",
      "Precision: 0.19\n",
      "F1 Score: 0.29\n"
     ]
    }
   ],
   "source": [
    "# Compute the accuracy of the final predictions\n",
    "correct_predictions = [prediction == label for prediction, label in zip(all_predictions, all_labels)]\n",
    "accuracy = sum(correct_predictions) / len(correct_predictions)\n",
    "print(f'Accuracy: {accuracy:.2f}')\n",
    "\n",
    "# Compute the recall of the final predictions\n",
    "true_positives = sum([prediction and label for prediction, label in zip(all_predictions, all_labels)])\n",
    "false_negatives = sum([not prediction and label for prediction, label in zip(all_predictions, all_labels)])\n",
    "recall = true_positives / (true_positives + false_negatives)\n",
    "print(f'Recall: {recall: .2f}')\n",
    "\n",
    "# Compute the precision of the final predictions\n",
    "true_positives = sum([prediction and label for prediction, label in zip(all_predictions, all_labels)])\n",
    "false_positives = sum([prediction and not label for prediction, label in zip(all_predictions, all_labels)])\n",
    "precision = true_positives / (true_positives + false_positives)\n",
    "print(f'Precision: {precision:.2f}')\n",
    "\n",
    "# F1 score\n",
    "f1 = 2 * (precision * recall) / (precision + recall)\n",
    "print(f'F1 Score: {f1:.2f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}