Merge branch 'chatgpt-database' into test

kairos/enums/assistant_data.py

@@ -22,6 +22,20 @@ class ASSISTANT_DATA:
                 },
             },
         },
+        {
+            "type": "function",
+            "function": {
+                "description": "Queries a MongoDB database cases collection for similar cases.",
+                "name": "find_similar_cases",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "case_id": {"type": "string"},
+                    },
+                    "required": ["case_id"],
+                },
+            },
+        },
     ]
 
     @classmethod
@@ -43,7 +57,7 @@ def instructions(cls, event_logs_db_structure=None, cases_db_structure=None, ins
       return cls.INSTRUCTIONS_BASE + instructions_database + instructions_examples
 
 
-    INSTRUCTIONS_BASE = """You are an AI assistant helping process analysts use Kairos, a process monitoring dashboard. Kairos uses three algorithms to generate prescriptions for business processes: NEXT_ACTIVITY: Predicts the next activity in a process using a KNN algorithm (always recommended). ALARM: Alerts users about a high probability of a negative outcome using a random forest algorithm. It's recommended to interneve in the case if the predicted probability exceeds a specified threshold. TREATMENT_EFFECT: Estimates the Conditional Average Treatment Effect (CATE) using the CausalLift algorithm, indicating how an intervention might alter the outcome of a case positively or negatively. If CATE score is positive, than the intervention is recommended. 
+    INSTRUCTIONS_BASE = """You are an AI assistant helping process analysts use Kairos, a process monitoring dashboard. Kairos uses three algorithms to generate prescriptions for business processes: NEXT_ACTIVITY: Predicts the next activity in a process using a KNN algorithm (always recommended). ALARM: Alerts users about a high probability of a negative outcome using a random forest algorithm. It's recommended to interneve in the case if the predicted probability exceeds a specified threshold. TREATMENT_EFFECT: Estimates the Conditional Average Treatment Effect (CATE) using the CausalLift algorithm, indicating how an intervention might alter the outcome of a case positively or negatively. If CATE score is positive, than the intervention is recommended. The CATE score is categorized into 'low', 'medium' and 'high' depending on these thresholds: low_threshold = mean_cate - THRESHOLD_FACTOR * std_dev_cate, high_threshold = mean_cate + THRESHOLD_FACTOR * std_dev_cate.
     
     The status of the prescriptions in the past (discarded or accepted) does not affect the status of the current (latest) prescriptions.
     
@@ -100,7 +114,8 @@ def instructions(cls, event_logs_db_structure=None, cases_db_structure=None, ins
       {date: '2023-11-01T15:52:40.354733',
       type: 'TREATMENT_EFFECT',
       output: {
-        cate: -0.5205 ,
+        cate: -0.5205,
+        cate_category: low,
         treatment: [[{ column: 'Activity', operator: 'EQUAL', value: 'O_SENT' }]] ,
         proba_if_treated: 0.323 ,
         proba_if_untreated: 0.232 
@@ -122,14 +137,16 @@ def instructions(cls, event_logs_db_structure=None, cases_db_structure=None, ins
     INSTRUCTIONS_EXAMPLES = """Here are some example questions and how to answer them:
 
 QUESTION: What is the size of the event log?	
-ANSWER: The event log consists of <number_of_cases> of cases.	
+ANSWER: The event log consists of <number_of_cases> of cases.
+FUNCTION: query_db	
 QUERY: {collection:'cases',
 aggregate:[{'$match':
 {'event_log_id':<EVENT_LOG_ID>}},
 {'$count': 'number_of_cases'}]}	
 STEPS: Run the query with function query_db to find the number of cases in this event log.
 
-QUESTION: Does this case have any recommendations?		
+QUESTION: Does this case have any recommendations?	
+FUNCTION: query_db	
 QUERY: {collection:'cases',
 aggregate:[{'$match':
 {_id:<CASE_ID>}},
@@ -141,6 +158,7 @@ def instructions(cls, event_logs_db_structure=None, cases_db_structure=None, ins
 
 QUESTION: What is the size, proportion, or distribution of the training data with given feature(s)/feature-value(s)?
 ANSWER: The train set consists of {number} of cases. The test set consists of {number} of cases. 
+FUNCTION: query_db
 QUERY: {collection:'files',
 aggregate:[{'$match':
 {'_id':<EVENT_LOG_ID>}},
@@ -156,6 +174,7 @@ def instructions(cls, event_logs_db_structure=None, cases_db_structure=None, ins
 
 QUESTION: What should be my action based on the recommendations?
 ANSWER: Based on the analysis, there are <COUNT OF PRESCRIPTIONS> possible recommendations. Recommendation <NAME> of type <TYPE> has {the highest probability / the highest CATE score} and therefore seems to be the best action. However, it is important that you assess the options carefully and make a final decision.  
+FUNCTION: query_db
 QUERY: {collection:'cases',
     aggregate:[
         {'$match': {'_id': <CASE_ID>}},
@@ -171,6 +190,7 @@ def instructions(cls, event_logs_db_structure=None, cases_db_structure=None, ins
 
 QUESTION: Why should I believe that the predictions are correct?  
 ANSWER: The accuracy of recommendations is on average <average_accuracy>. 
+FUNCTION: query_db
 QUERY: {collection:'cases',
     aggregate:[
         {'$match': {'event_log_id': <EVENT_LOG_ID>}},
@@ -192,6 +212,7 @@ def instructions(cls, event_logs_db_structure=None, cases_db_structure=None, ins
 
 QUESTION: How often does the system make mistakes?
 ANSWER: The average accuracy over the past instances is <average_accuracy>. 
+FUNCTION: query_db
 QUERY: {collection:'cases',
     aggregate:[
         {'$match': {'event_log_id': <EVENT_LOG_ID>}},
@@ -208,6 +229,7 @@ def instructions(cls, event_logs_db_structure=None, cases_db_structure=None, ins
 
 QUESTION: What kind of mistakes is the system likely to make? 
 ANSWER: The tool has an average accurcy of <average_accuracy>. 
+FUNCTION: query_db
 QUERY: {collection:'cases',
     aggregate:[
         {'$match': {'event_log_id': <EVENT_LOG_ID>}},
@@ -231,14 +253,30 @@ def instructions(cls, event_logs_db_structure=None, cases_db_structure=None, ins
 What kind of algorithms are used? 
 ANSWER: The tool provides three different recommendation types: next best activity, alarm and intervention. The next best activity is prescribed by an algorithm that employs a KNN-algorithm. The alarm recommendation is produced by random forest algorithm. The intervention is produced using Uplift Modeling package CasualLift to get the CATE and probability of outcome if the intervention is applied or not. 
 
-QUESTION: What features does the system consider? 
-ANSWER: The features that are used for training of the algorithms are: <case attributes that are used>   
+QUESTION: Why are cases <CASE_ID_1> and <CASE_ID_2> given the same recommendation?
+ANSWER: In the past, for a similar case <CASE_ID>, there were <number> recommendations prescribed. The recommendation <example> was applied. The case ended in <positive/negative> outcome. The cases <CASE_ID_1> and <CASE_ID_2> are similar in <case attributes / state / etc>.
+FUNCTION: query_db
 QUERY: {collection:'cases',
     aggregate:[
-        {'$match': {'_id': <CASE_ID>}},
-        {'$project': {'case_attributes': 1, '_id': 0}}
+        {'$match': {'_id': {"$in" : [<CASE_ID_1>, <CASE_ID_2>]}}}
     ]
 }  
-STEPS: Run the query using query_db function to get the attributes used for training. 
+STEPS: Run the query using query_db function to get the two cases and compare them.
 
+QUESTION: What are the alternative actions?
+or 
+What is the minimum change required for this case to get a different recommendation?
+or
+Were there similar cases in the past where similar recommendations led to positive outcomes?
+or 
+What is the necessary feature(s)/feature-value(s) present or absent to guarantee this recommendation?
+ANSWER: In the past, for a similar case <CASE_ID_2>, there were <number> recommendations prescribed. The recommendation <example> was applied. The case ended in <positive/negative> outcome. The current cases and case <CASE_ID_2> are similar in <case attributes / state / etc>.
+FUNCTION: find_similar_cases
+QUERY: {case_id: <CASE_ID>}  
+STEPS: Run the query using find_similar_cases function to get similar cases and compare them.
+
+QUESTION: What do the model performance metrics (accuracy, precision, recall) mean?
+ANSWER: Accuracy is calculated as the number of accurate predictions made by a model relative to the total predictions. In other words, accuracy shows how often the algorithm is correct overall. 
+Precision is calculated by dividing the number of correct positive predictions (true positives) by the total number of instances the algorithms predicted as positive (both true and false positives). In other words, precision shows how often the algorithm is correct when predicting the target class.
+Recall is calculated by dividing the number of true positives by the number of positive instances. The latter includes true positives (successfully identified cases) and false negative results (missed cases). Recall shows whether the algorithm can find all objects of the target class.
 """

kairos/models/db.py

@@ -16,4 +16,34 @@ def get_db():
 
 def query_db(collection, aggregate:list):
     db = LocalProxy(get_db)
-    return db[collection].aggregate(aggregate)
+    return db[collection].aggregate(aggregate)
+
+def find_similar_cases(case_id):
+    db = LocalProxy(get_db)
+
+    caseInstance = db.cases.find_one({"_id": case_id})
+    if caseInstance == None:
+        current_app.logger.error(f'No case found with id {case_id}.')
+        return f'No case found with id {case_id}.'
+
+    return list(db.cases.aggregate([
+        {
+            "$search": {
+                "index": "caseindex",
+                "moreLikeThis": {
+                    "like": caseInstance
+                }
+            }
+        },
+        { "$limit": 4 },
+        {
+            "$project": {
+                "_id": 1,
+                "case_completed": 1,
+                "event_log_id": 1,
+                "case_attributes": 1,
+                "case_performance": 1,
+                "activities": 1
+            }
+        }
+    ]))[1:]

kairos/utils/ai_assistant.py

@@ -1,5 +1,5 @@
 from openai import OpenAI
-from kairos.models.db import query_db
+from kairos.models.db import query_db, find_similar_cases
 from kairos.enums.assistant_data import ASSISTANT_DATA
 from flask import current_app
 from json import JSONDecodeError
@@ -53,7 +53,10 @@ def ask_ai(case_id,event_log_id,question):
             if run.required_action and run.required_action.type == "submit_tool_outputs":
                 tool_calls = run.required_action.submit_tool_outputs.tool_calls
                 if tool_calls:
-                    available_functions = {"query_db" : query_db}
+                    available_functions = {
+                        "query_db" : query_db,
+                        "find_similar_cases" : find_similar_cases
+                    }
                     tool_outputs = []
                     for tool_call in tool_calls:
 
@@ -62,28 +65,37 @@ def ask_ai(case_id,event_log_id,question):
                         function_to_call = available_functions[function_name]
 
                         function_args = convert_to_json(tool_call.function.arguments)
+
+                        if function_name == "query_db":
 
-                        collection = function_args.get('collection')
+                            collection = function_args.get('collection')
 
-                        if collection == None:
-                            current_app.logger.error('Collection field was null. Defaulting to base value.')
-                            collection = 'cases'
+                            if collection == None:
+                                current_app.logger.error('Collection field was null. Defaulting to base value.')
+                                collection = 'cases'
 
-                        aggregate = function_args.get('aggregate')
-                        query = function_args.get('query')
+                            aggregate = function_args.get('aggregate')
 
-                        if aggregate == None:
-                            current_app.logger.error(f'Aggregate field was null. Defaulting to base query. Query field: {query}')
-                            aggregate = [{"$match": {"_id": case_id}}]
+                            if aggregate == None:
+                                current_app.logger.error(f'Aggregate field was null. Defaulting to base query. Aggregate field: {aggregate}')
+                                aggregate = [{"$match": {"_id": case_id}}]
 
-                        current_app.logger.info(f'Running function {function_to_call} with args {collection,aggregate}')
+                            current_app.logger.info(f'Running function {function_to_call} with args {collection,aggregate}')
+
+                            function_response = function_to_call(
+                                collection=collection,
+                                aggregate=aggregate,
+                            )
+
+                        else:
+                            case_id_arg = function_args.get('case_id') or case_id
+                            current_app.logger.info(f'Running function {function_to_call} with args {case_id}')
+
+                            function_response = function_to_call(case_id=case_id_arg)
 
-                        function_response = function_to_call(
-                            collection=collection,
-                            aggregate=aggregate,
-                        )
                         function_response = str(list(function_response))
                         current_app.logger.info(f'Function respone: {function_response}')
+
                         tool_outputs.append({
                             "tool_call_id" : tool_call_id,
                             "output" : function_response