From a506f0c171153f86caebbe00023f02648d2326a1 Mon Sep 17 00:00:00 2001
From: Chan Jun Shern <chanjunshern@gmail.com>
Date: Wed, 22 May 2024 23:54:28 +0100
Subject: [PATCH] Add NotImplementedError if user tries to use a solver

---
 evals/elsuite/schelling_point/eval.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/evals/elsuite/schelling_point/eval.py b/evals/elsuite/schelling_point/eval.py
index 46d5371af1..78f6be68c6 100644
--- a/evals/elsuite/schelling_point/eval.py
+++ b/evals/elsuite/schelling_point/eval.py
@@ -9,6 +9,7 @@
 from evals.elsuite.schelling_point.prompts import sys_prompts_ci, sys_prompts_no_ci
 from evals.elsuite.schelling_point.utils import get_response
 from evals.eval import Eval
+from evals.solvers.solver import Solver
 
 
 class SchellingPoint(Eval):
@@ -22,6 +23,9 @@ def __init__(
         *args,
         **kwargs,
     ):
+        if any([isinstance(completion_fn, Solver) for completion_fn in completion_fns]):
+            raise NotImplementedError("SchellingPoint does not currently support Solvers")
+        
         super().__init__(completion_fns, *args, **kwargs)
 
         random.seed(seed)
@@ -34,7 +38,6 @@ def __init__(
 
         self.temperature = temperature
 
-        self.completion_fns = completion_fns
         if len(self.completion_fns) == 1:
             self.completion_fns = self.completion_fns * n_copies
         assert len(self.completion_fns) == n_copies, "Must provide n_copies completion_fns"