Skip to content

Commit

Permalink
check that placeholder {previous_responses} not in user prompt when c…
Browse files Browse the repository at this point in the history 
…heck combo insts
  • Loading branch information
@josh-ashkinaze
josh-ashkinaze committed Oct 15, 2024
1 parent fffa906 commit 5a418d2
Showing 1 changed file with 101 additions and 66 deletions.
167 changes: 101 additions & 66 deletions plurals/unit_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -598,24 +598,25 @@ def test_chain_with_different_combination_instructions(self):

def test_set_all_individual_agent_combination_instructions(self):
"""Set custom combo insts for all agents"""

task = "Generate argument for or against social media in 10 words."
agent1 = Agent(
task=task,
combination_instructions="Build on previous ideas: ${previous_responses}",
model="gpt-3.5-turbo",
kwargs={"max_tokens": 30}
kwargs={"max_tokens": 30},
)
agent2 = Agent(
task=task,
combination_instructions="Critique previous points: ${previous_responses}",
model="gpt-3.5-turbo",
kwargs={"max_tokens": 30}
kwargs={"max_tokens": 30},
)
agent3 = Agent(
task=task,
combination_instructions="Synthesize all points: ${previous_responses}",
model="gpt-3.5-turbo",
kwargs={"max_tokens": 30}
kwargs={"max_tokens": 30},
)

chain = Chain([agent1, agent2, agent3], task=task)
Expand All @@ -636,87 +637,71 @@ def test_set_all_individual_agent_combination_instructions(self):

self.assertEqual(3, len(chain.responses))

self.assertIn("Critique previous points:", agent2.prompts[0]["user"])
self.assertIn("Synthesize all points:", agent3.prompts[0]["user"])

def test_set_some_individual_agent_combination_instructions(self):
"""When combo insts set for some Agents, we use the set instructions for the relevant Agents
and structure combo instructions for other Agents"""
"""When combo insts set for some Agents, we use the set instructions for the relevant Agents and structure combo instructions for other Agents"""
task = "Generate argument for or against social media in 10 words."
agent1 = Agent(
task=task,
combination_instructions="Expand on previous responses: ${previous_responses}",
model="gpt-3.5-turbo",
kwargs={"max_tokens": 30}
kwargs={"max_tokens": 30},
)
agent2 = Agent(task=task, model="gpt-3.5-turbo", kwargs={"max_tokens": 30})
agent3 = Agent(task=task, model="gpt-3.5-turbo", kwargs={"max_tokens": 30})

chain = Chain([agent1, agent2, agent3], task=task, combination_instructions="chain")
chain = Chain(
[agent1, agent2, agent3], task=task, combination_instructions="jury"
)
chain.process()

self.assertEqual(
"Expand on previous responses: ${previous_responses}",
agent1.combination_instructions,
)
self.assertEqual(
DEFAULTS['combination_instructions']['chain'],
DEFAULTS["combination_instructions"]["jury"],
agent2.combination_instructions,
)
self.assertEqual(
DEFAULTS['combination_instructions']['chain'],
DEFAULTS["combination_instructions"]["jury"],
agent3.combination_instructions,
)

self.assertEqual(3, len(chain.responses))

self.assertIn("Reexamine your views", agent2.prompts[0]["user"])
self.assertIn("Reexamine your views", agent3.prompts[0]["user"])

def test_no_individual_agent_combination_instructions(self):
"""Make sure when no combo inst set, but set in structure, Agents get what is set in structure"""
task = "Generate argument for or against social media in 10 words."
agent1 = Agent(task=task, model="gpt-3.5-turbo", kwargs={"max_tokens": 30})
agent2 = Agent(task=task, model="gpt-3.5-turbo", kwargs={"max_tokens": 30})
agent3 = Agent(task=task, model="gpt-3.5-turbo", kwargs={"max_tokens": 30})

chain = Chain([agent1, agent2, agent3], task=task, combination_instructions="first_wave")
chain.process()

self.assertEqual(
DEFAULTS['combination_instructions']['first_wave'],
agent1.combination_instructions,
)
self.assertEqual(
DEFAULTS['combination_instructions']['first_wave'],
agent2.combination_instructions,
)
self.assertEqual(
DEFAULTS['combination_instructions']['first_wave'],
agent3.combination_instructions,
chain = Chain(
[agent1, agent2, agent3], task=task, combination_instructions="first_wave"
)

self.assertEqual(3, len(chain.responses))

def test_no_individual_agent_combination_instructions_default(self):
"""Make sure when no combo inst set every Agent gets default"""
task = "Generate argument for or against social media in 10 words."
agent1 = Agent(task=task, model="gpt-3.5-turbo", kwargs={"max_tokens": 30})
agent2 = Agent(task=task, model="gpt-3.5-turbo", kwargs={"max_tokens": 30})
agent3 = Agent(task=task, model="gpt-3.5-turbo", kwargs={"max_tokens": 30})

chain = Chain([agent1, agent2, agent3], task=task)
chain.process()

self.assertEqual(
DEFAULTS['combination_instructions']['default'],
DEFAULTS["combination_instructions"]["first_wave"],
agent1.combination_instructions,
)
self.assertEqual(
DEFAULTS['combination_instructions']['default'],
DEFAULTS["combination_instructions"]["first_wave"],
agent2.combination_instructions,
)
self.assertEqual(
DEFAULTS['combination_instructions']['default'],
DEFAULTS["combination_instructions"]["first_wave"],
agent3.combination_instructions,
)

self.assertEqual(3, len(chain.responses))

self.assertIn("Respect each other’s viewpoints", agent2.prompts[0]["user"])
self.assertIn("Respect each other’s viewpoints", agent3.prompts[0]["user"])

def test_chain_debate_instructions(self):
a2 = Agent(ideology="moderate", model=self.model)
Expand Down Expand Up @@ -1038,21 +1023,20 @@ def test_debate_with_different_combination_instructions(self):

self.assertIn("Counter the previous point:", agent2.prompts[0]["user"])


def test_set_all_individual_agent_combination_instructions(self):
"""Set custom combo insts for all agents"""
task = "Generate argument for or against social media in 10 words."
agent1 = Agent(
task=task,
combination_instructions="Respond to previous argument: ${previous_responses}",
model="gpt-3.5-turbo",
kwargs={"max_tokens": 30}
kwargs={"max_tokens": 30},
)
agent2 = Agent(
task=task,
combination_instructions="Counter the previous point: ${previous_responses}",
model="gpt-3.5-turbo",
kwargs={"max_tokens": 30}
kwargs={"max_tokens": 30},
)

debate = Debate([agent1, agent2], task=task)
Expand All @@ -1067,7 +1051,7 @@ def test_set_all_individual_agent_combination_instructions(self):
"Counter the previous point: ${previous_responses}",
)

self.assertEqual(len(debate.responses), 2)
self.assertNotIn("${previous_responses}", agent2.prompts[0]["user"])

self.assertIn("Counter the previous point:", agent2.prompts[0]["user"])

Expand All @@ -1079,7 +1063,7 @@ def test_set_some_individual_agent_combination_instructions(self):
task=task,
combination_instructions="Address the previous point: ${previous_responses}",
model="gpt-3.5-turbo",
kwargs={"max_tokens": 30}
kwargs={"max_tokens": 30},
)
agent2 = Agent(task=task, model="gpt-3.5-turbo", kwargs={"max_tokens": 30})

Expand All @@ -1090,23 +1074,37 @@ def test_set_some_individual_agent_combination_instructions(self):
"Address the previous point: ${previous_responses}",
agent1.combination_instructions,
)
self.assertEqual(DEFAULTS['combination_instructions']['debate'], agent2.combination_instructions)

self.assertEqual(len(debate.responses), 2)
self.assertEqual(
DEFAULTS["combination_instructions"]["debate"],
agent2.combination_instructions,
)
self.assertNotIn("${previous_responses}", agent2.prompts[0]["user"])
self.assertIn("in the third person", agent2.prompts[0]["user"])

def test_no_individual_agent_combination_instructions(self):
"""Make sure when no combo inst set, but set in structure, Agents get what is set in structure"""
task = "Generate argument for or against social media in 10 words."
agent1 = Agent(task=task, model="gpt-3.5-turbo", kwargs={"max_tokens": 30})
agent2 = Agent(task=task, model="gpt-3.5-turbo", kwargs={"max_tokens": 30})

debate = Debate([agent1, agent2], task=task, combination_instructions="first_wave")
debate = Debate(
[agent1, agent2], task=task, combination_instructions="first_wave"
)
debate.process()

self.assertEqual(DEFAULTS['combination_instructions']['first_wave'], agent1.combination_instructions)
self.assertEqual(DEFAULTS['combination_instructions']['first_wave'], agent2.combination_instructions)
self.assertEqual(
DEFAULTS["combination_instructions"]["first_wave"],
agent1.combination_instructions,
)
self.assertEqual(
DEFAULTS["combination_instructions"]["first_wave"],
agent2.combination_instructions,
)

self.assertEqual(len(debate.responses), 2)
self.assertNotIn("${previous_responses}", agent2.prompts[0]["user"])

self.assertIn("rational-critical debate", agent2.prompts[0]["user"])

def test_no_individual_agent_combination_instructions_default(self):
"""Make sure when no combo inst set every Agent gets default"""
Expand All @@ -1117,10 +1115,18 @@ def test_no_individual_agent_combination_instructions_default(self):
debate = Debate([agent1, agent2], task=task)
debate.process()

self.assertEqual(DEFAULTS['combination_instructions']['debate'], agent1.combination_instructions)
self.assertEqual(DEFAULTS['combination_instructions']['debate'], agent2.combination_instructions)
self.assertEqual(
DEFAULTS["combination_instructions"]["debate"],
agent1.combination_instructions,
)
self.assertEqual(
DEFAULTS["combination_instructions"]["debate"],
agent2.combination_instructions,
)
self.assertNotIn("${previous_responses}", agent2.prompts[0]["user"])

self.assertEqual(len(debate.responses), 2)
self.assertIn("third person", agent2.prompts[0]["user"])


class TestAgentStructures(unittest.TestCase):
Expand Down Expand Up @@ -1306,6 +1312,8 @@ def test_graph_with_different_combination_instructions(self):
self.assertIn("Prev2 to previous", agent2.prompts[0]["user"])
self.assertIn("Greetings to previous", agent3.prompts[0]["user"])

self.assertNotIn("${previous_responses}", agent2.prompts[0]["user"])
self.assertNotIn("${previous_responses}", agent3.prompts[0]["user"])

def test_set_all_individual_agent_combination_instructions(self):
"""Set custom combo insts for all agents"""
Expand All @@ -1314,19 +1322,19 @@ def test_set_all_individual_agent_combination_instructions(self):
task=task,
combination_instructions="Summarize previous insights: ${previous_responses}",
model="gpt-3.5-turbo",
kwargs={"max_tokens": 30}
kwargs={"max_tokens": 30},
)
agent2 = Agent(
task=task,
combination_instructions="Provide contrasting viewpoint: ${previous_responses}",
model="gpt-3.5-turbo",
kwargs={"max_tokens": 30}
kwargs={"max_tokens": 30},
)
agent3 = Agent(
task=task,
combination_instructions="Synthesize all perspectives: ${previous_responses}",
model="gpt-3.5-turbo",
kwargs={"max_tokens": 30}
kwargs={"max_tokens": 30},
)

agents = {"agent1": agent1, "agent2": agent2, "agent3": agent3}
Expand All @@ -1348,6 +1356,10 @@ def test_set_all_individual_agent_combination_instructions(self):
agent3.combination_instructions,
)

self.assertIn("Provide contrasting viewpoint:", agent2.prompts[0]["user"])
self.assertIn("Synthesize all perspectives:", agent3.prompts[0]["user"])
self.assertNotIn("${previous_responses}", agent2.prompts[0]["user"])
self.assertNotIn("${previous_responses}", agent3.prompts[0]["user"])
self.assertEqual(3, len(graph.responses))

def test_set_some_individual_agent_combination_instructions(self):
Expand All @@ -1358,30 +1370,36 @@ def test_set_some_individual_agent_combination_instructions(self):
task=task,
combination_instructions="Build on previous ideas: ${previous_responses}",
model="gpt-3.5-turbo",
kwargs={"max_tokens": 30}
kwargs={"max_tokens": 30},
)
agent2 = Agent(task=task, model="gpt-3.5-turbo", kwargs={"max_tokens": 30})
agent3 = Agent(task=task, model="gpt-3.5-turbo", kwargs={"max_tokens": 30})

agents = {"agent1": agent1, "agent2": agent2, "agent3": agent3}
edges = [("agent1", "agent2"), ("agent1", "agent3"), ("agent2", "agent3")]

graph = Graph(agents=agents, edges=edges, task=task, combination_instructions="voting")
graph = Graph(
agents=agents, edges=edges, task=task, combination_instructions="voting"
)
graph.process()

self.assertEqual(
"Build on previous ideas: ${previous_responses}",
agent1.combination_instructions,
)
self.assertEqual(
DEFAULTS['combination_instructions']['voting'],
DEFAULTS["combination_instructions"]["voting"],
agent2.combination_instructions,
)
self.assertEqual(
DEFAULTS['combination_instructions']['voting'],
DEFAULTS["combination_instructions"]["voting"],
agent3.combination_instructions,
)

self.assertIn("single and", agent2.prompts[0]["user"])
self.assertIn("single and", agent3.prompts[0]["user"])
self.assertNotIn("${previous_responses}", agent2.prompts[0]["user"])
self.assertNotIn("${previous_responses}", agent3.prompts[0]["user"])
self.assertEqual(3, len(graph.responses))

def test_no_individual_agent_combination_instructions(self):
Expand All @@ -1394,21 +1412,32 @@ def test_no_individual_agent_combination_instructions(self):
agents = {"agent1": agent1, "agent2": agent2, "agent3": agent3}
edges = [("agent1", "agent2"), ("agent1", "agent3"), ("agent2", "agent3")]

graph = Graph(agents=agents, edges=edges, task=task, combination_instructions="second_wave")
graph = Graph(
agents=agents,
edges=edges,
task=task,
combination_instructions="second_wave",
)
graph.process()

self.assertEqual(
DEFAULTS['combination_instructions']['second_wave'],
DEFAULTS["combination_instructions"]["second_wave"],
agent1.combination_instructions,
)
self.assertEqual(
DEFAULTS['combination_instructions']['second_wave'],
DEFAULTS["combination_instructions"]["second_wave"],
agent2.combination_instructions,
)
self.assertEqual(
DEFAULTS['combination_instructions']['second_wave'],
DEFAULTS["combination_instructions"]["second_wave"],
agent3.combination_instructions,
)

self.assertIn("Use empathy", agent2.prompts[0]["user"])
self.assertIn("Use empathy", agent3.prompts[0]["user"])

self.assertNotIn("${previous_responses}", agent2.prompts[0]["user"])
self.assertNotIn("${previous_responses}", agent3.prompts[0]["user"])
self.assertEqual(3, len(graph.responses))

def test_no_individual_agent_combination_instructions_default(self):
Expand All @@ -1425,18 +1454,24 @@ def test_no_individual_agent_combination_instructions_default(self):
graph.process()

self.assertEqual(
DEFAULTS['combination_instructions']['default'],
DEFAULTS["combination_instructions"]["default"],
agent1.combination_instructions,
)
self.assertEqual(
DEFAULTS['combination_instructions']['default'],
DEFAULTS["combination_instructions"]["default"],
agent2.combination_instructions,
)
self.assertEqual(
DEFAULTS['combination_instructions']['default'],
DEFAULTS["combination_instructions"]["default"],
agent3.combination_instructions,
)

self.assertIn("USE PREVIOUS", agent2.prompts[0]["user"])
self.assertIn("USE PREVIOUS", agent3.prompts[0]["user"])

self.assertNotIn("${previous_responses}", agent2.prompts[0]["user"])
self.assertNotIn("${previous_responses}", agent3.prompts[0]["user"])

self.assertEqual(3, len(graph.responses))

def test_dict_to_list_conversion(self):
Expand Down

0 comments on commit 5a418d2

Please sign in to comment.