created common set of loggers and included readmefile for logging

flo_ai/__init__.py

@@ -5,4 +5,4 @@
 from flo_ai.state.flo_session import FloSession
 from flo_ai.retrievers.flo_retriever import FloRagBuilder
 from flo_ai.common.flo_logger import get_logger
-from flo_ai.common.flo_callback_handler import FloCallbackHandler
+from flo_ai.common.flo_langchain_logger import FloLangchainLogger

flo_ai/common/README.md

@@ -0,0 +1,67 @@
+# Understanding Log Levels
+
+FLO uses standard Python logging levels to indicate the severity of logged messages. Here are the common levels used (from least to most severe):
+
+DEBUG: Detailed information for debugging purposes.
+INFO: Informational messages about the normal operation of the system.
+WARNING: Potential issues or unexpected conditions.
+ERROR: Errors that may have caused the system to malfunction.
+CRITICAL: Critical errors that have caused the system to crash.
+By adjusting the log level, you can control how much information is logged and the verbosity of the output.
+
+Controlling Log Levels
+FLO provides multiple ways to control log levels:
+
+1. Environment Variables:
+
+Export environment variables to set the log level for specific components before running the application:
+
+FLO_LOG_LEVEL_COMMON: Controls the level for the "CommonLogs" logger.
+FLO_LOG_LEVEL_BUILDER: Controls the level for the "BuilderLogs" logger.
+FLO_LOG_LEVEL_SESSION: Controls the level for the "SessionLogs" logger.
+
+Example:
+
+export FLO_LOG_LEVEL_COMMON=DEBUG
+export FLO_LOG_LEVEL_BUILDER=INFO
+export FLO_LOG_LEVEL_SESSION=WARNING
+
+
+2. FloSession Creation:
+
+When creating a FloSession object, you can specify the desired log level:
+
+session = FloSession(llm, log_level="DEBUG")
+
+ This session will log messages at DEBUG level and above.
+
+3. Flo Instance Creation:
+
+Similar to FloSession, you can set the log level when creating a Flo instance:
+
+flo = Flo.build(session, yaml_config, log_level="DEBUG")
+
+This Flo instance will inherit the specified log level.
+
+4. Global Log Level Change (Runtime):
+
+You can dynamically change the global log level at runtime using the set_global_log_level function from flo_ai.common.logging_config:
+
+from flo_ai.common.logging_config import set_global_log_level
+
+set_global_log_level("DEBUG")  # Set the global log level to DEBUG
+
+This will affect all logging throughout the application.
+
+5. Specific Logger Level Change (Runtime):
+
+If you need to adjust the level for a specific logger, use the set_log_level method of the FloLogger class:
+
+from flo_ai.common.logging_config import FloLogger
+
+FloLogger.set_log_level("COMMON", "DEBUG")  # Set COMMON logger to DEBUG level
+
+
+Environment variables: Use these for setting default levels without modifying code, useful in different deployment environments.
+Object creation: This approach allows setting specific levels for individual sessions or Flo instances.
+Runtime changes: Use these methods for dynamic adjustments during program execution.

flo_ai/common/__init__.py

@@ -1,4 +0,0 @@
-from .flo_logger import get_logger, FloLogger
-from .flo_callback_handler import FloCallbackHandler
-
-__all__ = ['get_logger', 'FloLogger', 'FloCallbackHandler']

flo_ai/common/flo_langchain_logger.py

@@ -0,0 +1,47 @@
+from typing import Any, Dict, List, Union
+from langchain.callbacks.base import BaseCallbackHandler
+from langchain.schema import AgentAction, AgentFinish, LLMResult
+from flo_ai.common.flo_logger import get_logger
+
+class FloLangchainLogger(BaseCallbackHandler):
+    def __init__(self, logger_name: str = "FloLangChainLogger", log_level: str = "INFO"):
+        self.logger = get_logger(logger_name, log_level)
+
+    def on_llm_start(self, serialized: Dict[str, Any], prompts: List[str], **kwargs: Any) -> None:
+        self.logger.info(f"onLLMStart: {prompts}")
+
+    def on_llm_new_token(self, token: str, **kwargs: Any) -> None:
+        self.logger.debug(f"onNewToken: {token}")
+
+    def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None:
+        self.logger.info(f"onLLMEnd: {response.generations}")
+
+    def on_llm_error(self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any) -> None:
+        self.logger.error(f"onLLMError: {error}")
+
+    def on_chain_start(self, serialized: Dict[str, Any], inputs: Dict[str, Any], **kwargs: Any) -> None:
+        self.logger.info(f"onChainStart: {inputs}")
+
+    def on_chain_end(self, outputs: Dict[str, Any], **kwargs: Any) -> None:
+        self.logger.info(f"onChainEnd: {outputs}")
+
+    def on_chain_error(self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any) -> None:
+        self.logger.error(f"onChainError: {error}")
+
+    def on_tool_start(self, serialized: Dict[str, Any], input_str: str, **kwargs: Any) -> None:
+        self.logger.info(f"onToolStart: {input_str}")
+
+    def on_tool_end(self, output: str, **kwargs: Any) -> None:
+        self.logger.info(f"onToolEnd: {output}")
+
+    def on_tool_error(self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any) -> None:
+        self.logger.error(f"onToolError: {error}")
+
+    def on_text(self, text: str, **kwargs: Any) -> None:
+        self.logger.info(f"onText: {text}")
+
+    def on_agent_action(self, action: AgentAction, **kwargs: Any) -> Any:
+        self.logger.info(f"onAgentAction: {action.tool} - {action.tool_input}")
+
+    def on_agent_finish(self, finish: AgentFinish, **kwargs: Any) -> None:
+        self.logger.info(f"onAgentFinish: {finish.return_values}")

flo_ai/common/flo_logger.py

@@ -1,78 +1,46 @@
-import logging
 import os
+import logging
 from logging.handlers import RotatingFileHandler
-from typing import Optional
-
-
-class FloLogger:
-    _instances = {}
+from typing import Dict, Optional
 
-    def __new__(cls, name: str, level: str = "INFO", use_file: bool = True, file_path: str = "flo.log", max_bytes: int = 1048576):
-        """
-        Custom new method to check for existing instance.
-        """
-        if name not in cls._instances:
-            instance = super().__new__(cls)  
-            cls._instances[name] = instance
-        return cls._instances[name]
+class FloLoggerUtil(logging.Logger):
+    def __init__(self, name: str, level: str = "INFO", use_file: bool = False, file_path: str = None, max_bytes: int = 1048576):
+        super().__init__(name, level)
+        self.setLevel(level)
 
-    def __init__(self, name: str, level: str = "INFO", use_file: bool = True, file_path: str = "flo.log", max_bytes: int = 1048576):
-
-        self.logger = logging.getLogger(name)
-        if not self.logger.handlers:
-            self.set_level(level)
-            self._setup_handler(use_file, file_path, max_bytes)
-
-    def _setup_handler(self, use_file: bool, file_path: str, max_bytes: int):
-        handler = logging.StreamHandler()
         formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
-        handler.setFormatter(formatter)
-        self.logger.addHandler(handler)
+
+        console_handler = logging.StreamHandler()
+        console_handler.setFormatter(formatter)
+        self.addHandler(console_handler)
 
         if use_file:
-            log_file_path = os.path.join(os.path.dirname(__file__), file_path)
-            file_handler = RotatingFileHandler(
-                log_file_path, 
-                maxBytes=max_bytes, 
-            )
+            file_handler = RotatingFileHandler(file_path or f"{name}.log", maxBytes=max_bytes)
             file_handler.setFormatter(formatter)
-            self.logger.addHandler(file_handler)
-
-    def set_level(self, level):
-        if isinstance(level, str):
-            level = getattr(logging, level.upper(), logging.INFO)
-        self.logger.setLevel(level)
+            self.addHandler(file_handler)
 
-    def debug(self, message: str):
-        self.logger.debug(message)
-
-    def info(self, message: str):
-        self.logger.info(message)
-
-    def warning(self, message: str):
-        self.logger.warning(message)
-
-    def error(self, message: str):
-        self.logger.error(message)
-
-    def critical(self, message: str):
-        self.logger.critical(message)
-
-
-def get_logger(name: str, level: Optional[str] = 'INFO', use_file: bool = True, file_path: str = "flo.log") -> FloLogger:
-    """
-    Creates a FloLogger instance with optional file logging and caching.
-
-    Args:
-        name: The name of the logger.
-        level: The logging level (default: INFO).
-        use_file: Whether to log to a file (default: False).
-        file_path: The path to the log file (default: "flo.log").
-        max_bytes: The maximum size of the log file before it gets rotated (default: 1MB).
-
-    Returns:
-        A FloLogger instance.
-    """
-
-    logger = FloLogger(name, level, use_file, file_path)
-    return logger
+class FloLogger:
+    _loggers = {}
+
+    @classmethod
+    def get_logger(cls, name: str, level: str = None, use_file: bool = False, file_path: str = None, max_bytes: int = 1048576) -> FloLoggerUtil:
+        if name not in cls._loggers:
+            level = level or os.environ.get(f"FLO_LOG_LEVEL_{name.upper()}", "INFO")
+            cls._loggers[name] = FloLoggerUtil(name, level, use_file, file_path, max_bytes)
+        return cls._loggers[name]
+
+    @classmethod
+    def set_log_level(cls, name: str, level: str):
+        if name in cls._loggers:
+            cls._loggers[name].setLevel(level)
+
+def get_logger(name: str, level: str = None, use_file: bool = False, file_path: str = None, max_bytes: int = 1048576) -> FloLoggerUtil:
+    return FloLogger.get_logger(name, level, use_file, file_path, max_bytes)
+
+common_logger = get_logger("COMMON")
+builder_logger = get_logger("BUILDER")
+session_logger = get_logger("SESSION")
+
+def set_global_log_level(level: str):
+    for name in ["COMMON", "BUILDER", "SESSION"]:
+        FloLogger.set_log_level(name, level)

flo_ai/core.py

@@ -7,18 +7,22 @@
 )
 from flo_ai.state.flo_session import FloSession
 from flo_ai.models.flo_executable import ExecutableFlo
-from flo_ai.common.flo_logger import FloLogger, get_logger
+from flo_ai.common.flo_logger import common_logger, builder_logger, FloLogger
 
 class Flo:
 
     def __init__(self,
                  session: FloSession,
-                 config: FloRoutedTeamConfig) -> None:
+                 config: FloRoutedTeamConfig,
+                 log_level: str = "INFO") -> None:
         self.config = config
         self.session = session
         self.runnable: ExecutableFlo = build_supervised_team(session, config)
-        self.logger = session.logger
-        self.callback_handler = session.callback_handler
+
+        FloLogger.set_log_level("COMMON", log_level)
+        FloLogger.set_log_level("BUILDER", log_level)
+        self.logger = common_logger
+        self.langchain_logger = session.langchain_logger
         self.logger.info(f"Flo instance created for session {session.session_id}")
 
     def stream(self, query, config = None) -> Iterator[Union[dict[str, Any], Any]]:
@@ -30,10 +34,10 @@ def invoke(self, query, config = None) -> Iterator[Union[dict[str, Any], Any]]:
         return self.runnable.invoke(query, config)
 
     @staticmethod
-    def build(session: FloSession, yaml: str):
-        logger = get_logger("Flo.build", session.logger.logger.level)
-        logger.info("Building Flo instance from YAML")
-        return Flo(session, to_supervised_team(yaml))
+    def build(session: FloSession, yaml: str, log_level: str = "INFO"):
+        FloLogger.set_log_level("BUILDER", log_level)
+        builder_logger.info("Building Flo instance from YAML")
+        return Flo(session, to_supervised_team(yaml), log_level)
 
     def draw(self, xray=True):
         return self.runnable.draw(xray)

flo_ai/state/flo_session.py

@@ -1,18 +1,17 @@
 import uuid
 from langchain_core.language_models import BaseLanguageModel
 from langchain_core.tools import BaseTool
-from flo_ai.common.flo_logger import FloLogger, get_logger
-from flo_ai.common.flo_callback_handler import FloCallbackHandler
+from flo_ai.common.flo_logger import session_logger, FloLogger, get_logger
+from flo_ai.common.flo_langchain_logger import FloLangchainLogger
 from typing import Optional
 
 class FloSession:
     def __init__(self, 
                  llm: BaseLanguageModel, 
                  loop_size: int = 2, 
                  max_loop: int = 3, 
-                 log_level: str = "DEBUG",
-                 custom_logger: Optional[FloLogger] = None,
-                 custom_callback_handler: Optional[FloCallbackHandler] = None) -> None:
+                 log_level: str = "INFO",
+                 custom_langchainlog_handler: Optional[FloLangchainLogger] = None) -> None:
         self.session_id = str(uuid.uuid4())
         self.llm = llm
         self.tools = dict()
@@ -21,9 +20,10 @@ def __init__(self,
         self.pattern_series = dict()
         self.loop_size: int = loop_size
         self.max_loop: int = max_loop
-        self.logger = custom_logger or get_logger(f"FloSession-{self.session_id}", log_level)
-        self.callback_handler = custom_callback_handler or FloCallbackHandler(f"FloCallback-{self.session_id}", log_level)
+        FloLogger.set_log_level("SESSION", log_level)
+        self.logger = session_logger
         self.logger.info(f"New FloSession created with ID: {self.session_id}")
+        self.langchain_logger = custom_langchainlog_handler or FloLangchainLogger(f"FloLangChainLogger-{self.session_id}", log_level)
 
     def register_tool(self, name: str, tool: BaseTool):
         self.tools[name] = tool