From a2d53fcc647714256bce18322b1deab25bc1ef79 Mon Sep 17 00:00:00 2001 From: RockChinQ <1010553892@qq.com> Date: Fri, 6 Oct 2023 09:16:46 +0000 Subject: [PATCH 1/4] feat: log to database --- free_one_api/impls/app.py | 49 +++++++++++++++++++++++++++ free_one_api/impls/database/sqlite.py | 25 +++++++++++++- free_one_api/impls/forward/mgr.py | 5 +++ free_one_api/impls/log.py | 20 +++++++++++ free_one_api/models/database/db.py | 14 ++++++++ 5 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 free_one_api/impls/log.py diff --git a/free_one_api/impls/app.py b/free_one_api/impls/app.py index 63da2e1..5b42f80 100644 --- a/free_one_api/impls/app.py +++ b/free_one_api/impls/app.py @@ -1,6 +1,8 @@ import os import sys import asyncio +import logging +import colorlog import yaml @@ -21,6 +23,8 @@ from .adapter import hugchat from .adapter import qianwen +from . import log + class Application: """Application instance.""" @@ -61,6 +65,14 @@ async def run(self): await self.router.serve(loop) +log_colors_config = { + 'DEBUG': 'green', # cyan white + 'INFO': 'white', + 'WARNING': 'yellow', + 'ERROR': 'red', + 'CRITICAL': 'cyan', +} + default_config = { "database": { "type": "sqlite", @@ -79,6 +91,9 @@ async def run(self): }, "web": { "frontend_path": "./web/dist/", + }, + "logging": { + "debug": False, } } @@ -93,7 +108,31 @@ async def make_application(config_path: str) -> Application: config = {} with open(config_path, "r") as f: config = yaml.load(f, Loader=yaml.FullLoader) + + # logging + logging_level = logging.INFO + + if 'logging' in config and 'debug' in config['logging'] and config['logging']['debug']: + logging_level = logging.DEBUG + if 'DEBUG' in os.environ and os.environ['DEBUG'] == 'true': + logging_level = logging.DEBUG + + terminal_out = logging.StreamHandler() + + terminal_out.setLevel(logging_level) + terminal_out.setFormatter(colorlog.ColoredFormatter( + "[%(asctime)s.%(msecs)03d] %(log_color)s%(filename)s (%(lineno)d) - [%(levelname)s] : " + "%(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + log_colors=log_colors_config, + )) + + for handler in logging.getLogger().handlers: + logging.getLogger().removeHandler(handler) + + logging.getLogger().addHandler(terminal_out) + # make database manager from .database import sqlite as sqlitedb @@ -104,6 +143,14 @@ async def make_application(config_path: str) -> Application: dbmgr = dbmgr_cls_mapping[config['database']['type']](config['database']) await dbmgr.initialize() + # database handler + dblogger = log.SQLiteHandler(dbmgr) + + dblogger.setLevel(logging_level) + dblogger.setFormatter(logging.Formatter("[%(asctime)s.%(msecs)03d] %(pathname)s (%(lineno)d) - [%(levelname)s] :\n%(message)s")) + + logging.getLogger().addHandler(dblogger) + # make channel manager from .channel import mgr as chanmgr @@ -171,5 +218,7 @@ async def make_application(config_path: str) -> Application: watchdog=wdmgr, ) + logging.info("Application initialized.") + return app \ No newline at end of file diff --git a/free_one_api/impls/database/sqlite.py b/free_one_api/impls/database/sqlite.py index c76839f..3d9ea94 100644 --- a/free_one_api/impls/database/sqlite.py +++ b/free_one_api/impls/database/sqlite.py @@ -27,6 +27,14 @@ ) """ +log_table_sql = """ +CREATE TABLE IF NOT EXISTS log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp INTEGER NOT NULL, + content TEXT NOT NULL +) +""" + class SQLiteDB(dbmod.DatabaseInterface): @@ -38,8 +46,9 @@ async def initialize(self): async with aiosqlite.connect(self.db_path) as db: await db.execute(channel_table_sql) await db.execute(key_table_sql) + await db.execute(log_table_sql) await db.commit() - + async def get_channel(self, channel_id: int) -> channel.Channel: async with aiosqlite.connect(self.db_path) as db: async with db.execute("SELECT * FROM channel WHERE id = ?", (channel_id,)) as cursor: @@ -134,3 +143,17 @@ async def delete_key(self, key_id: int) -> None: async with aiosqlite.connect(self.db_path) as db: await db.execute("DELETE FROM apikey WHERE id = ?", (key_id,)) await db.commit() + + async def insert_log(self, timestamp: int, content: str) -> None: + async with aiosqlite.connect(self.db_path) as db: + await db.execute("INSERT INTO log (timestamp, content) VALUES (?, ?)", ( + timestamp, + content, + )) + await db.commit() + + async def select_logs(self, time_range: tuple[int, int]) -> list[tuple[int, str]]: + async with aiosqlite.connect(self.db_path) as db: + async with db.execute("SELECT * FROM log WHERE timestamp >= ? AND timestamp <= ?", time_range) as cursor: + rows = await cursor.fetchall() + return [(row[1], row[2]) for row in rows] diff --git a/free_one_api/impls/forward/mgr.py b/free_one_api/impls/forward/mgr.py index e5f366a..b8ccfbd 100644 --- a/free_one_api/impls/forward/mgr.py +++ b/free_one_api/impls/forward/mgr.py @@ -2,6 +2,7 @@ import json import string import random +import logging import quart @@ -165,6 +166,10 @@ async def query( id_suffix += chan.adapter.__class__.__name__[:10] id_suffix += "".join(random.choices(string.ascii_letters+string.digits, k=29-len(id_suffix))) + query_info_str = f"query: path={path}, model={req.model}, id_suffix={id_suffix}, channel_name={chan.name}, channel_adpater={chan.adapter.__class__.__name__}" + + logging.info(query_info_str) + if req.stream: return await self.__stream_query(chan, req, id_suffix) else: diff --git a/free_one_api/impls/log.py b/free_one_api/impls/log.py new file mode 100644 index 0000000..b43fdb1 --- /dev/null +++ b/free_one_api/impls/log.py @@ -0,0 +1,20 @@ +import logging +import asyncio + +from ..models.database import db + + +class SQLiteHandler(logging.Handler): + """SQLite logging handler.""" + + dbmgr: db.DatabaseInterface + + def __init__(self, dbmgr: db.DatabaseInterface, level=logging.NOTSET): + super().__init__(level) + self.dbmgr = dbmgr + + def emit(self, record: logging.LogRecord): + """Emit a record.""" + loop = asyncio.get_running_loop() + + loop.create_task(self.dbmgr.insert_log(record.created, record.getMessage())) diff --git a/free_one_api/models/database/db.py b/free_one_api/models/database/db.py index 84d8128..120f45b 100644 --- a/free_one_api/models/database/db.py +++ b/free_one_api/models/database/db.py @@ -50,3 +50,17 @@ async def update_key(self, key: apikey.FreeOneAPIKey) -> None: async def delete_key(self, key_id: int) -> None: """Delete a key.""" return + + @abc.abstractmethod + async def insert_log(self, timestamp: int, content: str) -> None: + """Insert a log.""" + return + + @abc.abstractmethod + async def select_logs(self, time_range: tuple[int, int]) -> list[tuple[int, str]]: + """Select logs. + + Args: + time_range: (start, end) + """ + return From ac18a9d627643053fd83ce04cc2e56eba55a8c08 Mon Sep 17 00:00:00 2001 From: RockChinQ <1010553892@qq.com> Date: Fri, 6 Oct 2023 10:34:22 +0000 Subject: [PATCH 2/4] feat: show logs --- ...ag.yaml => build-publish-release-tag.yaml} | 2 +- design/API.md | 3 + free_one_api/impls/database/sqlite.py | 17 ++- free_one_api/impls/router/api.py | 36 ++++- free_one_api/models/database/db.py | 17 ++- web/src/App.vue | 3 + web/src/components/Log.vue | 129 ++++++++++++++++++ web/src/mock/index.js | 60 ++++++++ 8 files changed, 262 insertions(+), 5 deletions(-) rename .github/workflows/{build-release-tag.yaml => build-publish-release-tag.yaml} (98%) create mode 100644 web/src/components/Log.vue diff --git a/.github/workflows/build-release-tag.yaml b/.github/workflows/build-publish-release-tag.yaml similarity index 98% rename from .github/workflows/build-release-tag.yaml rename to .github/workflows/build-publish-release-tag.yaml index 7cb333c..0cf464a 100644 --- a/.github/workflows/build-release-tag.yaml +++ b/.github/workflows/build-publish-release-tag.yaml @@ -8,7 +8,7 @@ on: jobs: publish-release-docker-image: runs-on: ubuntu-latest - name: Build image + name: Build release image steps: - name: Checkout diff --git a/design/API.md b/design/API.md index f4eaed8..ea235ce 100644 --- a/design/API.md +++ b/design/API.md @@ -18,6 +18,9 @@ - GET /raw/ - POST /create - DEL /revoke/ + - /log + - GET /list + - page:int capacity:int ## Entities diff --git a/free_one_api/impls/database/sqlite.py b/free_one_api/impls/database/sqlite.py index 3d9ea94..1779272 100644 --- a/free_one_api/impls/database/sqlite.py +++ b/free_one_api/impls/database/sqlite.py @@ -152,8 +152,21 @@ async def insert_log(self, timestamp: int, content: str) -> None: )) await db.commit() - async def select_logs(self, time_range: tuple[int, int]) -> list[tuple[int, str]]: + async def select_logs(self, time_range: tuple[int, int]) -> list[tuple[int, int, str]]: async with aiosqlite.connect(self.db_path) as db: async with db.execute("SELECT * FROM log WHERE timestamp >= ? AND timestamp <= ?", time_range) as cursor: rows = await cursor.fetchall() - return [(row[1], row[2]) for row in rows] + return [(row[0], row[1], row[2]) for row in rows] + + async def select_logs_page(self, capacity: int, page: int) -> list[tuple[int, int, str]]: + async with aiosqlite.connect(self.db_path) as db: + async with db.execute("SELECT * FROM log ORDER BY id DESC LIMIT ? OFFSET ?", (capacity, capacity * page)) as cursor: + rows = await cursor.fetchall() + return [(row[0], row[1], row[2]) for row in rows] + + async def get_logs_amount(self) -> int: + async with aiosqlite.connect(self.db_path) as db: + async with db.execute("SELECT COUNT(*) FROM log") as cursor: + row = await cursor.fetchone() + return row[0] + \ No newline at end of file diff --git a/free_one_api/impls/router/api.py b/free_one_api/impls/router/api.py index be890b2..11cbe27 100644 --- a/free_one_api/impls/router/api.py +++ b/free_one_api/impls/router/api.py @@ -257,4 +257,38 @@ async def key_revoke(key_id: int): return quart.jsonify({ "code": 1, "message": str(e), - }) \ No newline at end of file + }) + + @self.api("/log/list", ["GET"], auth=True) + async def list_logs(): + try: + data = quart.request.args + + capacity = int(data.get("capacity", 20)) + page = int(data.get("page", 0)) + + amount = await self.dbmgr.get_logs_amount() + + logs = await self.dbmgr.select_logs_page(capacity, page) + + return quart.jsonify({ + "code": 0, + "message": "ok", + "data": { + "amount": amount, + "logs": [ + { + "id": log[0], + "timestamp": log[1], + "content": log[2], + } for log in logs + ], + }, + }) + except Exception as e: + import traceback + traceback.print_exc() + return quart.jsonify({ + "code": 1, + "message": str(e), + }) diff --git a/free_one_api/models/database/db.py b/free_one_api/models/database/db.py index 120f45b..3c94b3c 100644 --- a/free_one_api/models/database/db.py +++ b/free_one_api/models/database/db.py @@ -57,10 +57,25 @@ async def insert_log(self, timestamp: int, content: str) -> None: return @abc.abstractmethod - async def select_logs(self, time_range: tuple[int, int]) -> list[tuple[int, str]]: + async def select_logs(self, time_range: tuple[int, int]) -> list[tuple[int, int, str]]: """Select logs. Args: time_range: (start, end) """ return + + @abc.abstractmethod + async def select_logs_page(self, capacity: int, page: int) -> list[tuple[int, int, str]]: + """Select logs, sort descending by timestamp. + + Args: + capacity: number of logs per page + page: page number + """ + return + + @abc.abstractmethod + async def get_logs_amount(self) -> int: + """Get the amount of logs.""" + return diff --git a/web/src/App.vue b/web/src/App.vue index eb06968..59fd4ff 100644 --- a/web/src/App.vue +++ b/web/src/App.vue @@ -2,6 +2,7 @@ import Home from './components/Home.vue'; import Channel from './components/Channel.vue'; import APIKey from './components/APIKey.vue'; +import Log from './components/Log.vue'; import { setPassword, getPassword, clearPassword, checkPassword } from './common/account'; import { ElMessageBox, ElMessage } from 'element-plus'; @@ -54,6 +55,7 @@ function logout(){
Home
Channels
API Keys
+
Logs
+
diff --git a/web/src/components/Log.vue b/web/src/components/Log.vue new file mode 100644 index 0000000..9075cb6 --- /dev/null +++ b/web/src/components/Log.vue @@ -0,0 +1,129 @@ + + + + + diff --git a/web/src/mock/index.js b/web/src/mock/index.js index b604126..55023fd 100644 --- a/web/src/mock/index.js +++ b/web/src/mock/index.js @@ -31,3 +31,63 @@ Mock.mock("/check_password", "post", { "code": 0, "message": "ok", }) + +Mock.mock(/\/log\/list.*/, "get", { + "code": 0, + "message": "ok", + "data": { + "page_count": 10, + "logs": [ + { + "id": 1, + "timestamp": 1612345678, + "content": "test", + }, + { + "id": 2, + "timestamp": 1612345678, + "content": "test", + }, + { + "id": 3, + "timestamp": 1612345678, + "content": "test", + }, + { + "id": 4, + "timestamp": 1612345678, + "content": "test", + }, + { + "id": 5, + "timestamp": 1612345678, + "content": "test", + }, + { + "id": 6, + "timestamp": 1612345678, + "content": "test", + }, + { + "id": 7, + "timestamp": 1612345678, + "content": "test", + }, + { + "id": 8, + "timestamp": 1612345678, + "content": "test", + }, + { + "id": 9, + "timestamp": 1612345678, + "content": "test", + }, + { + "id": 10, + "timestamp": 1612345678, + "content": "test", + } + ] + } +}) From e842dec5b0e6940a08d141c088a70b22d8842336 Mon Sep 17 00:00:00 2001 From: Junyan Qin <1010553892@qq.com> Date: Fri, 6 Oct 2023 11:07:30 +0000 Subject: [PATCH 3/4] feat: logs panel --- free_one_api/impls/router/api.py | 3 ++- web/src/components/Log.vue | 15 ++++++++++++--- web/src/mock/index.js | 2 +- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/free_one_api/impls/router/api.py b/free_one_api/impls/router/api.py index 11cbe27..36c9485 100644 --- a/free_one_api/impls/router/api.py +++ b/free_one_api/impls/router/api.py @@ -268,6 +268,7 @@ async def list_logs(): page = int(data.get("page", 0)) amount = await self.dbmgr.get_logs_amount() + page_count = (amount + capacity - 1) // capacity logs = await self.dbmgr.select_logs_page(capacity, page) @@ -275,7 +276,7 @@ async def list_logs(): "code": 0, "message": "ok", "data": { - "amount": amount, + "page_count": page_count, "logs": [ { "id": log[0], diff --git a/web/src/components/Log.vue b/web/src/components/Log.vue index 9075cb6..2a99674 100644 --- a/web/src/components/Log.vue +++ b/web/src/components/Log.vue @@ -13,9 +13,10 @@ const page_count = ref(0); const keyContainerWidth = ref("1000px"); function refreshLogs(){ - axios.get("/log/list", { + console.log("refreshing logs") + axios.get("/api/log/list", { params: { - capacity: 20, + capacity: 10, page: page.value } }).then((response) => { @@ -23,7 +24,14 @@ function refreshLogs(){ if (response.data.code === 0){ page_count.value = response.data.data.page_count; - logs.value = response.data.data.logs; + + var logs_raw = response.data.data.logs; + + // 把所有timestamp转换成可读的格式 + for (let i = 0; i < logs_raw.length; i++){ + logs_raw[i].time = new Date(logs_raw[i].timestamp * 1000).toLocaleString(); + } + logs.value = logs_raw; }else{ ElNotification({ message: "Failed: "+res.data.message, @@ -67,6 +75,7 @@ onresize = () => { + diff --git a/web/src/mock/index.js b/web/src/mock/index.js index 55023fd..20a0bf2 100644 --- a/web/src/mock/index.js +++ b/web/src/mock/index.js @@ -32,7 +32,7 @@ Mock.mock("/check_password", "post", { "message": "ok", }) -Mock.mock(/\/log\/list.*/, "get", { +Mock.mock(/\/api\/log\/list.*/, "get", { "code": 0, "message": "ok", "data": { From 2cb3243ccc5618b13239f913d52bffebe24ce366 Mon Sep 17 00:00:00 2001 From: Junyan Qin <1010553892@qq.com> Date: Fri, 6 Oct 2023 11:08:05 +0000 Subject: [PATCH 4/4] perf: log format --- free_one_api/impls/forward/mgr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/free_one_api/impls/forward/mgr.py b/free_one_api/impls/forward/mgr.py index b8ccfbd..f816cf2 100644 --- a/free_one_api/impls/forward/mgr.py +++ b/free_one_api/impls/forward/mgr.py @@ -166,7 +166,7 @@ async def query( id_suffix += chan.adapter.__class__.__name__[:10] id_suffix += "".join(random.choices(string.ascii_letters+string.digits, k=29-len(id_suffix))) - query_info_str = f"query: path={path}, model={req.model}, id_suffix={id_suffix}, channel_name={chan.name}, channel_adpater={chan.adapter.__class__.__name__}" + query_info_str = f"type=query, path={path}, model={req.model}, id_suffix={id_suffix}, channel_name={chan.name}, channel_adpater={chan.adapter.__class__.__name__}" logging.info(query_info_str)