refactors the naming of constitutions to princeples

adds methods for comparison data generation
adds example principles for comparison

ConstitutionalAiTuning/principles_loader/__init__.py

@@ -0,0 +1,4 @@
+# Initialize the principles_loader package.
+
+# Import and expose functions or classes, if necessary.
+from .principles_loader import load_principles

ConstitutionalAiTuning/principles_loader/principles_loader.py

@@ -0,0 +1,18 @@
+import json
+
+def load_principles(file_path):
+    """
+    Loads a constitutioal principles file (JSON format) and returns the parsed content.
+
+    :param file_path: The path to the JSON file containing the principles.
+    :return: A dictionary representing the loaded principles.
+    """
+    try:
+        with open(file_path, 'r') as f:
+            principles = json.load(f)
+    except FileNotFoundError:
+        raise FileNotFoundError(f"Principles file not found at: {file_path}")
+    except json.JSONDecodeError:
+        raise ValueError(f"Invalid JSON format in file: {file_path}")
+
+    return principles

ConstitutionalAiTuning/prompting/prompt_template.py

@@ -2,59 +2,80 @@
 
 class PromptTemplate:
     """
-    A class to generate prompt answer pairs for fine-tuning LLMs using the constitutional AI approach.
+    A class for generating prompts to improve an initial model answer using one of the provided critique and revsion principles or to select the better of two generated model answers based on one of the provided comparison principles. The used prompt templates are based on the ones used in Constitutional AI approach by Anthropic.
 
     Attributes:
         user_prompt (str): The prompt for which the LLM answer will be improved.
         initial_answer (str): The initial answer generated by the LLM.
         critique (str): The critique of the initial answer.
         revision (str): The revised answer based on the critique.
-        constitution_instructions (dict): Constitution instructions including the information for generating prompts.
-        critique_request (str): Specific request for critique generation.
-        revision_request (str): Specific request for revision generation.
+        critique_principle (str): Specific request for critique generation.
+        revision_principle (str): Specific request for revision generation.
+        principles (dict): Constitutional principles for answer improvement and comparison.
     """
 
     def __init__(
         self,
         user_prompt,
         initial_answer=None,
-        critique_request=None,
+        critique_principle=None,
         critique=None,
-        revision_request=None,
+        revision_principle=None,
         revision=None,
-        constitution_instructions=None
+        comparison_answer_1=None,
+        comparison_answer_2=None,
+        comparison_principle=None,
+        chain_of_thought=None,
+        selected_answer=None,
+        principles=None
     ):
         """
         Initializes the PromptTemplate class with the provided arguments.
 
         Args:
             user_prompt (str): The prompt for which the LLM answer will be improved.
             initial_answer (str, optional): The initial answer. Defaults to None.
-            critique_request (str, optional): Specific request for critique generation. Defaults to None.
+            critique_principle (str, optional): Specific request for critique generation. Defaults to None.
             critique (str, optional): The critique of the initial answer. Defaults to None.
-            revision_request (str, optional): Specific request for revision generation. Defaults to None.
+            revision_principle (str, optional): Specific request for revision generation. Defaults to None.
             revision (str, optional): The revised answer. Defaults to None.
-            constitution_instructions (dict, optional): Constitution instructions for generating prompts. Defaults to a default structure.
+            comparison_answer_1 (str, optional): The first answer for comparison. Defaults to None.
+            comparison_answer_2 (str, optional): The second answer for comparison. Defaults to None.
+            comparison_principle (str, optional): Specific request for comparison generation. Defaults to None.
+            chain_of_thought (str, optional): The generated chain of thought for the comparison. Defaults to None.
+            selected_answer (str, optional): The selected answer from the comparison. Defaults to None.
+            principles (dict, optional): Constitution instructions for generating prompts. Defaults to a default structure.
         """
         self.user_prompt = user_prompt
         self.initial_answer = initial_answer
         self.critique = critique
         self.revision = revision
-        self.constitution_instructions = constitution_instructions or {
+        self.comparison_answer_1 = comparison_answer_1
+        self.comparison_answer_2 = comparison_answer_2
+        self.chain_of_thought = chain_of_thought
+        self.selected_answer = selected_answer
+        self.principles = principles or {
             "system_message_user_prompt": "",
             "system_message_critique": "",
             "system_message_revision": "",
-            "critique_revision_requests": [],
-            "examples": [],
+            "critique_revision_principles": [],
+            "critique_revision_few_shots": [],
+            "comparison_principles": [],
+            "comparison_few_shots": [],
         }
 
-        if critique_request or revision_request or len(self.constitution_instructions["critique_revision_requests"]) == 0:
-            self.critique_request = critique_request
-            self.revision_request = revision_request
+        if critique_principle or revision_principle or len(self.principles["critique_revision_principles"]) == 0:
+            self.critique_principle = critique_principle
+            self.revision_principle = revision_principle
         else:
-            critique_revision_request = random.choice(self.constitution_instructions["critique_revision_requests"])
-            self.critique_request = critique_revision_request["critique"]
-            self.revision_request = critique_revision_request["revision"]
+            critique_revision_principle = random.choice(self.principles["critique_revision_principles"])
+            self.critique_principle = critique_revision_principle["critique"]
+            self.revision_principle = critique_revision_principle["revision"]
+
+        if comparison_principle or len(self.principles["comparison_principles"]) == 0:
+            self.comparison_principle = comparison_principle
+        else:
+            self.comparison_principle = random.choice(self.principles["comparison_principles"])
 
     def generate_system_prompt(self, message):
         """
@@ -72,28 +93,31 @@ def generate_system_prompt(self, message):
         }]
 
     def generate_system_prompt_initial_answer(self):
-        return self.generate_system_prompt(self.constitution_instructions["system_message_user_prompt"])
+        return self.generate_system_prompt(self.principles["system_message_user_prompt"])
 
     def generate_system_prompt_critique(self):
-        return self.generate_system_prompt(self.constitution_instructions["system_message_critique"])
+        return self.generate_system_prompt(self.principles["system_message_critique"])
 
     def generate_system_prompt_revision(self):
-        return self.generate_system_prompt(self.constitution_instructions["system_message_revision"])
+        return self.generate_system_prompt(self.principles["system_message_revision"])
 
-    def generate_initial_answer_prompt(self):
+    def generate_initial_answer_prompt(self, include_system_prompt=True):
         input_prompt = [{
             "role": "user",
             "content": self.user_prompt,
         }]
-        return self.generate_system_prompt_initial_answer() + input_prompt
-
+        if include_system_prompt:
+            return self.generate_system_prompt_initial_answer() + input_prompt
+        else:
+            return input_prompt
+
     def generate_few_shot_critique_prompt(self):
         few_shot_critique_prompt = []
-        for example in self.constitution_instructions["examples"]:
+        for example in self.principles["critique_revision_few_shots"]:
             # Format the conversation history as a single string
             conversation_history = f"Human: {example.get('input', '')}\n" + \
                                    f"Assistant: {example.get('initial_answer', '')}\n\n" + \
-                                   f"---\nCritiqueRequest: {example.get('critique_request', '')}"
+                                   f"---\nCritiqueRequest: {example.get('critique_principle', '')}"
             user_input = {
                 "role": "user",
                 "content": conversation_history
@@ -111,20 +135,20 @@ def generate_critique_prompt(self):
             "role": "user",
             "content": f"Human: {self.user_prompt}\n"
                        f"Assistant: {self.initial_answer}\n\n"
-                       f"---\nCritiqueRequest: {self.critique_request}",
+                       f"---\nCritiqueRequest: {self.critique_principle}",
         }]
         # Combine prefix, few shot examples, and input
         return self.generate_system_prompt_critique() + self.generate_few_shot_critique_prompt() + input_prompt
 
     def generate_few_shot_revision_prompt(self):
         few_shot_revision_prompt = []
-        for example in self.constitution_instructions["examples"]:
+        for example in self.principles["critique_revision_few_shots"]:
             # Format the complete conversation history as a single string
             conversation_history = f"Human: {example.get('input', '')}\n" + \
                                    f"Assistant: {example.get('initial_answer', '')}\n" + \
-                                   f"---\nCritiqueRequest: {example.get('critique_request', '')}\n" + \
+                                   f"---\nCritiqueRequest: {example.get('critique_principle', '')}\n" + \
                                    f"Critique: {example.get('critique', '')}\n\n" + \
-                                   f"---\nRevisionRequest: {example.get('revision_request', '')}"
+                                   f"---\nRevisionRequest: {example.get('revision_principle', '')}"
             user_input = {
                 "role": "user",
                 "content": conversation_history
@@ -142,20 +166,83 @@ def generate_revision_prompt(self):
             "role": "user",
             "content": f"Human: {self.user_prompt}\n"
                        f"Assistant: {self.initial_answer}\n\n"
-                       f"---\nCritiqueRequest: {self.critique_request}"
+                       f"---\nCritiqueRequest: {self.critique_principle}"
                        f"Critique: {self.critique}\n\n"
-                       f"---\nRevisionRequest: {self.revision_request}",
+                       f"---\nRevisionRequest: {self.revision_principle}",
         }]
         # Combine prefix, few shot examples, and input
         return self.generate_system_prompt_revision() + self.generate_few_shot_revision_prompt() + input_prompt
 
+    def generate_few_shot_comparison_prompt(self):
+        few_shot_comparison_prompt = []
+        for example in self.principles["comparison_few_shots"]:
+            # Format the conversation history as a single string
+            single_shot_comparison_prompt = [
+                {
+                    "role": "user",
+                    "content": f"{example.get('task', '')}"
+                },
+                {
+                    "role": "assistant",
+                    "content": f"{example.get('chain_of_thought', '')}"
+                },
+                {
+                    "role": "user",
+                    "content": f"{example.get('selected_answer', '')}"
+                }
+            ]
+            few_shot_comparison_prompt.extend(single_shot_comparison_prompt)
+        return few_shot_comparison_prompt
+
+    def generate_chain_of_thought_comparison_prompt(self, include_system_prompt=False):
+        input_prompt = [
+            {
+                "role": "user",
+                "content": f"Consider the following conversation between a human (H) and an assistant (A):\n\nH: {self.user_prompt}\n\n{self.comparison_principle}\n(A)[[[{self.comparison_answer_1}]]]\n(B) [[[{self.comparison_answer_2}]]]"
+            },
+            {
+                "role": "assistant",
+                "content": f"Let's think step by step:\n"
+            }
+        ]
+        # Combine system prompt (if applicable) few shot examples and input
+        if include_system_prompt:
+            return self.generate_system_prompt() + self.generate_few_shot_comparison_prompt() + input_prompt
+        else:
+            return self.generate_few_shot_comparison_prompt() + input_prompt
+
+    def generate_selected_answer_comparison_prompt(self, include_system_prompt=False):
+        input_prompt = [
+            {
+                "role": "user",
+                "content": f"Consider the following conversation between a human (H) and an assistant (A):\n\nH: {self.user_prompt}\n\n{self.comparison_principle}\n(A)[[[{self.comparison_answer_1}]]]\n(B) [[[{self.comparison_answer_2}]]]"
+            },
+            {
+                "role": "assistant",
+                "content": f"Let's think step by step:\n{self.chain_of_thought}"
+            },
+            {
+                "role": "user",
+                "content": f"So the answer is: "
+            },
+        ]
+        if include_system_prompt:
+            return self.generate_system_prompt() + self.generate_few_shot_comparison_prompt() + input_prompt
+        else:
+            return self.generate_few_shot_comparison_prompt() + input_prompt
+
     def get_history(self):
-        return {
+        history = {
             "user_prompt": self.user_prompt,
             "initial_answer": self.initial_answer,
-            "critique_request": self.critique_request,
+            "critique_principle": self.critique_principle,
             "critique": self.critique,
-            "revision_request": self.revision_request,
+            "revision_principle": self.revision_principle,
             "revision": self.revision,
+            "comparison_answer_1": self.comparison_answer_1,
+            "comparison_answer_2": self.comparison_answer_2,
+            "comparison_principle": self.comparison_principle,
+            "chain_of_thought": self.chain_of_thought,
+            "selected_answer": self.selected_answer,
         }
-
+        return {k: v for k, v in history.items() if v}

examples/chat_usage.py

@@ -6,16 +6,18 @@
 system_message = "You are a tutor that always responds in the Socratic style. You *never* give the student the answer, but try to ask just the right question to help them learn to think for themselves. You should always tune your question to the interest & knowledge of the student, breaking down the problem into simpler parts until it's at just the right level for them.\nAlways ask just ONE question for each user message. DO NOT ask multiple questions at once."
 
 # Initialize the ModelInteractor with a Hugging Face model and poyyibly a system message
-# interactor = ModelInteractor(hf_model="HuggingFaceH4/zephyr-7b-beta", hf_api_key=HF_API_KEY, endpoint_url="https://m07124gncoa31nmm.eu-west-1.aws.endpoints.huggingface.cloud", system_message=None)
-interactor = ModelInteractor(hf_model="HuggingFaceH4/zephyr-7b-beta", hf_api_key=HF_API_KEY, system_message=system_message)
+# interactor = ModelInteractor(hf_model="HuggingFaceH4/zephyr-7b-beta", hf_api_key=HF_API_KEY, system_message=system_message)
+
+interactor = ModelInteractor(hf_model="HuggingFaceH4/zephyr-7b-beta", hf_api_key=HF_API_KEY, endpoint_url="https://dl6sir63fshnm5rr.eu-west-1.aws.endpoints.huggingface.cloud", system_message=None)
+
 
 # Start a new chat
-user_prompt = "Why does a ball thrown in the air come back down?"
+user_prompt = "How come that rainbows exist?"
 assistant_response = interactor.chat_with_model(user_prompt)
 print(f"Assistant: {assistant_response}")
 
 # Continue the chat
-user_prompt = "Why is not floating in the air?"
+user_prompt = "Yes, I have seen it before. It has different colors."
 assistant_response = interactor.chat_with_model(user_prompt)
 print(f"Assistant: {assistant_response}")