Merge pull request #17 from RockChinQ/feat/audit-module

Feat: log module

.github/workflows/build-release-tag.yaml → .github/workflows/build-publish-release-tag.yaml

@@ -8,7 +8,7 @@ on:
 jobs:
   publish-release-docker-image:
     runs-on: ubuntu-latest
-    name: Build image
+    name: Build release image
 
     steps:
       - name: Checkout

design/API.md

@@ -18,6 +18,9 @@
         - GET /raw/<id:int>
         - POST /create
         - DEL /revoke/<id:int>
+    - /log
+        - GET /list
+            - page:int capacity:int
 
 ## Entities
 

free_one_api/impls/app.py

@@ -1,6 +1,8 @@
 import os
 import sys
 import asyncio
+import logging
+import colorlog
 
 import yaml
 
@@ -21,6 +23,8 @@
 from .adapter import hugchat
 from .adapter import qianwen
 
+from . import log
+
 
 class Application:
     """Application instance."""
@@ -61,6 +65,14 @@ async def run(self):
 
         await self.router.serve(loop)
 
+log_colors_config = {
+    'DEBUG': 'green',  # cyan white
+    'INFO': 'white',
+    'WARNING': 'yellow',
+    'ERROR': 'red',
+    'CRITICAL': 'cyan',
+}
+
 default_config = {
     "database": {
         "type": "sqlite",
@@ -79,6 +91,9 @@ async def run(self):
     },
     "web": {
         "frontend_path": "./web/dist/",
+    },
+    "logging": {
+        "debug": False,
     }
 }
 
@@ -93,7 +108,31 @@ async def make_application(config_path: str) -> Application:
     config = {}
     with open(config_path, "r") as f:
         config = yaml.load(f, Loader=yaml.FullLoader)
+
+    # logging
+    logging_level = logging.INFO
+
+    if 'logging' in config and 'debug' in config['logging'] and config['logging']['debug']:
+        logging_level = logging.DEBUG
 
+    if 'DEBUG' in os.environ and os.environ['DEBUG'] == 'true':
+        logging_level = logging.DEBUG
+
+    terminal_out = logging.StreamHandler()
+
+    terminal_out.setLevel(logging_level)
+    terminal_out.setFormatter(colorlog.ColoredFormatter(
+        "[%(asctime)s.%(msecs)03d] %(log_color)s%(filename)s (%(lineno)d) - [%(levelname)s] : "
+            "%(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+        log_colors=log_colors_config,
+    ))
+
+    for handler in logging.getLogger().handlers:
+        logging.getLogger().removeHandler(handler)
+
+    logging.getLogger().addHandler(terminal_out)    
+
     # make database manager
     from .database import sqlite as sqlitedb
 
@@ -104,6 +143,14 @@ async def make_application(config_path: str) -> Application:
     dbmgr = dbmgr_cls_mapping[config['database']['type']](config['database'])
     await dbmgr.initialize()
 
+    # database handler
+    dblogger = log.SQLiteHandler(dbmgr)
+
+    dblogger.setLevel(logging_level)
+    dblogger.setFormatter(logging.Formatter("[%(asctime)s.%(msecs)03d] %(pathname)s (%(lineno)d) - [%(levelname)s] :\n%(message)s"))
+
+    logging.getLogger().addHandler(dblogger)
+
     # make channel manager
     from .channel import mgr as chanmgr
 
@@ -171,5 +218,7 @@ async def make_application(config_path: str) -> Application:
         watchdog=wdmgr,
     )
 
+    logging.info("Application initialized.")
+
     return app
 

free_one_api/impls/database/sqlite.py

@@ -27,6 +27,14 @@
 )
 """
 
+log_table_sql = """
+CREATE TABLE IF NOT EXISTS log (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    timestamp INTEGER NOT NULL,
+    content TEXT NOT NULL
+)
+"""
+
 
 class SQLiteDB(dbmod.DatabaseInterface):
 
@@ -38,8 +46,9 @@ async def initialize(self):
         async with aiosqlite.connect(self.db_path) as db:
             await db.execute(channel_table_sql)
             await db.execute(key_table_sql)
+            await db.execute(log_table_sql)
             await db.commit()
-            
+
     async def get_channel(self, channel_id: int) -> channel.Channel:
         async with aiosqlite.connect(self.db_path) as db:
             async with db.execute("SELECT * FROM channel WHERE id = ?", (channel_id,)) as cursor:
@@ -134,3 +143,30 @@ async def delete_key(self, key_id: int) -> None:
         async with aiosqlite.connect(self.db_path) as db:
             await db.execute("DELETE FROM apikey WHERE id = ?", (key_id,))
             await db.commit()
+
+    async def insert_log(self, timestamp: int, content: str) -> None:
+        async with aiosqlite.connect(self.db_path) as db:
+            await db.execute("INSERT INTO log (timestamp, content) VALUES (?, ?)", (
+                timestamp,
+                content,
+            ))
+            await db.commit()
+
+    async def select_logs(self, time_range: tuple[int, int]) -> list[tuple[int, int, str]]:
+        async with aiosqlite.connect(self.db_path) as db:
+            async with db.execute("SELECT * FROM log WHERE timestamp >= ? AND timestamp <= ?", time_range) as cursor:
+                rows = await cursor.fetchall()
+                return [(row[0], row[1], row[2]) for row in rows]
+
+    async def select_logs_page(self, capacity: int, page: int) -> list[tuple[int, int, str]]:
+        async with aiosqlite.connect(self.db_path) as db:
+            async with db.execute("SELECT * FROM log ORDER BY id DESC LIMIT ? OFFSET ?", (capacity, capacity * page)) as cursor:
+                rows = await cursor.fetchall()
+                return [(row[0], row[1], row[2]) for row in rows]
+
+    async def get_logs_amount(self) -> int:
+        async with aiosqlite.connect(self.db_path) as db:
+            async with db.execute("SELECT COUNT(*) FROM log") as cursor:
+                row = await cursor.fetchone()
+                return row[0]
+

free_one_api/impls/forward/mgr.py

@@ -2,6 +2,7 @@
 import json
 import string
 import random
+import logging
 
 import quart
 
@@ -165,6 +166,10 @@ async def query(
         id_suffix += chan.adapter.__class__.__name__[:10]
         id_suffix += "".join(random.choices(string.ascii_letters+string.digits, k=29-len(id_suffix)))
 
+        query_info_str = f"type=query, path={path}, model={req.model}, id_suffix={id_suffix}, channel_name={chan.name}, channel_adpater={chan.adapter.__class__.__name__}"
+
+        logging.info(query_info_str)
+
         if req.stream:
             return await self.__stream_query(chan, req, id_suffix)
         else:

free_one_api/impls/log.py

@@ -0,0 +1,20 @@
+import logging
+import asyncio
+
+from ..models.database import db
+
+
+class SQLiteHandler(logging.Handler):
+    """SQLite logging handler."""
+
+    dbmgr: db.DatabaseInterface
+
+    def __init__(self, dbmgr: db.DatabaseInterface, level=logging.NOTSET):
+        super().__init__(level)
+        self.dbmgr = dbmgr
+
+    def emit(self, record: logging.LogRecord):
+        """Emit a record."""
+        loop = asyncio.get_running_loop()
+
+        loop.create_task(self.dbmgr.insert_log(record.created, record.getMessage()))

free_one_api/impls/router/api.py

@@ -257,4 +257,39 @@ async def key_revoke(key_id: int):
                 return quart.jsonify({
                     "code": 1,
                     "message": str(e),
-                })
+                })
+
+        @self.api("/log/list", ["GET"], auth=True)
+        async def list_logs():
+            try:
+                data = quart.request.args
+
+                capacity = int(data.get("capacity", 20))
+                page = int(data.get("page", 0))
+
+                amount = await self.dbmgr.get_logs_amount()
+                page_count = (amount + capacity - 1) // capacity
+
+                logs = await self.dbmgr.select_logs_page(capacity, page)
+
+                return quart.jsonify({
+                    "code": 0,
+                    "message": "ok",
+                    "data": {
+                        "page_count": page_count,
+                        "logs": [
+                            {
+                                "id": log[0],
+                                "timestamp": log[1],
+                                "content": log[2],
+                            } for log in logs
+                        ],
+                    },
+                })
+            except Exception as e:
+                import traceback
+                traceback.print_exc()
+                return quart.jsonify({
+                    "code": 1,
+                    "message": str(e),
+                })

free_one_api/models/database/db.py

@@ -50,3 +50,32 @@ async def update_key(self, key: apikey.FreeOneAPIKey) -> None:
     async def delete_key(self, key_id: int) -> None:
         """Delete a key."""
         return
+
+    @abc.abstractmethod
+    async def insert_log(self, timestamp: int, content: str) -> None:
+        """Insert a log."""
+        return
+
+    @abc.abstractmethod
+    async def select_logs(self, time_range: tuple[int, int]) -> list[tuple[int, int, str]]:
+        """Select logs.
+        
+        Args:
+            time_range: (start, end)
+        """
+        return
+
+    @abc.abstractmethod
+    async def select_logs_page(self, capacity: int, page: int) -> list[tuple[int, int, str]]:
+        """Select logs, sort descending by timestamp.
+        
+        Args:
+            capacity: number of logs per page
+            page: page number
+        """
+        return
+
+    @abc.abstractmethod
+    async def get_logs_amount(self) -> int:
+        """Get the amount of logs."""
+        return

web/src/App.vue

@@ -2,6 +2,7 @@
 import Home from './components/Home.vue';
 import Channel from './components/Channel.vue';
 import APIKey from './components/APIKey.vue';
+import Log from './components/Log.vue';
 import { setPassword, getPassword, clearPassword, checkPassword } from './common/account';
 
 import { ElMessageBox, ElMessage } from 'element-plus';
@@ -54,6 +55,7 @@ function logout(){
         <div class="tab_btn flex_container" @click="switchTab('home')">Home</div>
         <div class="tab_btn flex_container" @click="switchTab('channel')">Channels</div>
         <div class="tab_btn flex_container" @click="switchTab('apikey')">API Keys</div>
+        <div class="tab_btn flex_container" @click="switchTab('logs')">Logs</div>
         <div id="login_info">
             <el-button :type="getPassword()==''?'success':'danger'"
             @click="getPassword()==''?showLoginDialog():logout()"
@@ -67,6 +69,7 @@ function logout(){
         <Home v-if="currentTab === 'home'"></Home>
         <Channel v-if="currentTab === 'channel'"></Channel>
         <APIKey v-if="currentTab === 'apikey'"></APIKey>
+        <Log v-if="currentTab === 'logs'"></Log>
     </div>
 </template>