v2.7.1 add ModelQueryAdaptor, alter model adaptor query functions, fi…

…x async issues on mysql, fix issues on schema query recursion, support database using in orm.Schema, add tests for mysql & postgresql database

.github/workflows/test.yaml

@@ -56,6 +56,37 @@ jobs:
 
     name: ${{ matrix.name }}
     runs-on: ${{ matrix.os }}
+
+    services:
+      postgres:
+        image: postgres:17
+        ports:
+          - 5432:5432
+        env:
+          POSTGRES_USER: test_user
+          POSTGRES_PASSWORD: test-tmp-password
+          POSTGRES_DB: utilmeta_test_db
+        options: >-
+          --health-cmd="pg_isready -U test_user"
+          --health-interval=5s
+          --health-timeout=5s
+          --health-retries=5
+
+      mysql:
+        image: mysql:8
+        ports:
+          - 3306:3306
+        env:
+          MYSQL_ROOT_PASSWORD: mysqlrootpassword
+          MYSQL_USER: test_user
+          MYSQL_PASSWORD: test-tmp-password
+          MYSQL_DATABASE: utilmeta_test_db
+        options: >-
+          --health-cmd="mysqladmin -u test_user -p${MYSQL_PASSWORD} ping"
+          --health-interval=5s
+          --health-timeout=5s
+          --health-retries=5
+
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
@@ -78,6 +109,32 @@ jobs:
       - name: Run lint
         run: |
           flake8 utilmeta --count --select=E9,F63,F7,F82 --show-source --statistics
+
+      - name: Wait for PostgreSQL
+        run: |
+          for i in {1..30}; do
+            if pg_isready -h localhost -p 5432 -U test_user; then
+              echo "PostgreSQL is ready!"
+              break
+            fi
+            echo "Waiting for PostgreSQL to be ready..."
+            sleep 5
+          done
+          sudo -i -u postgres psql -c 'create utilmeta_test_ops_db;'
+          sudo -i -u postgres psql -c 'grant all privileges on database utilmeta_test_ops_db to test_user;'
+
+      - name: Wait for MySQL
+        run: |
+          for i in {1..30}; do
+            if mysqladmin -h localhost -P 3306 -u test_user -p${MYSQL_PASSWORD} ping --silent; then
+              echo "MySQL is ready!"
+              break
+            fi
+            echo "Waiting for MySQL to be ready..."
+            sleep 5
+          done
+          mysql -h localhost -P 3306 -u test_user -p${MYSQL_PASSWORD} -e "CREATE DATABASE utilmeta_test_ops_db;"
+
       - name: Run tests
         run: |
           pytest tests --cov=./utilmeta

docs/en/guide/ops.md

@@ -235,7 +235,7 @@ The additional specified OpenAPI document will be integrated with the automatica
 
 ## Connect to UtilMeta Platform
 
-UtilMeta provides a management platform for the observation and management operations of the API service operation and maintenance management system: [ UtilMeta API Service Management Platform ](https://ops.utilmeta.com) you can enter the platform to connect and manage your own UtilMeta service, view API documents, data, logs and monitoring.
+UtilMeta provides a platform for the observation and management operations of the API services: [UtilMeta Platform ](https://ops.utilmeta.com) you can enter the platform to connect and manage your UtilMeta service (or other services with supported frameworks), view API documents, data, logs and monitoring.
 ### Connect Local Node
 
 If you have introduced the Operations configuration, successfully run the local service, and see the following prompt

docs/en/tutorials/user-auth.md

@@ -166,7 +166,7 @@ We open the `user/api.py` and write
 
 ```python
 from datetime import datetime
-from utilmeta.core import api, orm
+from utilmeta.core import api, orm, request
 from utilmeta.utils import exceptions
 from .models import User
 from . import auth

docs/zh/guide/ops.md

@@ -238,8 +238,8 @@ service.use(Operations(
 
 除了在公网部署和提供访问的 API 服务外，我们有时也需要管理内网集群中的 API 服务，比如公司内网的内部服务，这些服务没有公开的 IP 地址或访问 URL，管理这些内网服务需要设置内网集群中的公网代理，部署一个代理服务节点进行内网穿透与服务注册
 
-UtilMeta 已经提供了一个开源的代理服务 utilmeta-proxy：
-开源仓库：[https://github.com/utilmeta/utilmeta-proxy](https://github.com/utilmeta/utilmeta-proxy)
+UtilMeta 已经提供了一个开源的代理服务 utilmeta-proxy：[https://github.com/utilmeta/utilmeta-proxy](https://github.com/utilmeta/utilmeta-proxy)
+
 
 在 Operations 配置中，可以使用 `proxy` 参数配置代理服务节点的地址与设置
 

docs/zh/tutorials/user-auth.md

@@ -171,7 +171,7 @@ user_config = auth.User(
 
 ```python
 from datetime import datetime
-from utilmeta.core import api, orm
+from utilmeta.core import api, orm, request
 from utilmeta.utils import exceptions
 from .models import User
 from . import auth

examples/mini_blog/client.py

@@ -28,6 +28,7 @@ async def article_get(
         self, id: int = request.QueryParam(required=True)
     ) -> responses.article_get_response[200]: pass
 
+
 import httpx
 client = APIClient(base_url="http://127.0.0.1:8080", backend=httpx)
 # >>> resp = await client.article_get(id=1)

examples/user_auth/user/api.py

@@ -36,7 +36,7 @@ def get_session_data(self):
 @session_config.plugin
 class UserAPI(api.API):
     @api.post
-    def login(self, data: LoginSchema):
+    def login(self, data: LoginSchema = request.Body):
         user = user_config.login(
             request=self.request,
             ident=data.username,
@@ -47,7 +47,7 @@ def login(self, data: LoginSchema):
         return UserSchema.init(user)
 
     @api.post
-    def signup(self, data: LoginSchema):
+    def signup(self, data: LoginSchema = request.Body):
         if User.objects.filter(username=data.username).exists():
             raise exceptions.BadRequest('Username exists')
         data.save()

tests/conftest.py

@@ -3,7 +3,6 @@
 import time
 import subprocess
 import signal
-import warnings
 import pytest
 import psutil
 from utilmeta import UtilMeta
@@ -41,7 +40,40 @@
 #         cleanup_on_sigterm()
 
 
-def setup_service(name, backend: str = None, async_param: Union[list, bool] = True, orm: str = None):
+@pytest.fixture(params=('default', 'mysql', 'postgresql'), scope="module")
+def db_using(request):
+    return request.param
+
+
+def get_operations_db():
+    engine = os.environ.get('UTILMETA_OPERATIONS_DATABASE_ENGINE') or 'sqlite3'
+    from utilmeta.core.orm import Database
+    if engine == 'mysql':
+        return Database(
+            engine=engine,
+            name='utilmeta_test_ops_db',
+            host='127.0.0.1',
+            port=3306,
+            user='test_user',
+            password='test-tmp-password',
+        )
+    elif engine == 'postgresql':
+        return Database(
+            engine=engine,
+            name='utilmeta_test_ops_db',
+            host='127.0.0.1',
+            port=5432,
+            user='test_user',
+            password='test-tmp-password',
+        )
+    return Database(engine='sqlite3', name='operations_db')
+
+
+def setup_service(
+    name,
+    backend: str = None,
+    async_param: Union[list, bool] = True,
+):
     """
     If a list of params is provided, each param will not across and will execute in order
     for every ConfigParam, params inside are consider crossing, will enumerate every possible combination
@@ -366,7 +398,9 @@ def service_process(service: UtilMeta):
     return service_process
 
 
-def make_cmd_process(file: Union[str, Path], *argv, cwd: Union[str, Path] = None, port: int = None):
+def make_cmd_process(file: Union[str, Path], *argv,
+                     cwd: Union[str, Path] = None,
+                     port: int = None):
     if not os.path.isabs(file):
         file = os.path.join(TEST_PATH, file)
     if not os.path.exists(file):
@@ -377,11 +411,13 @@ def make_cmd_process(file: Union[str, Path], *argv, cwd: Union[str, Path] = None
     # raise Exception(str(cmd))
 
     @pytest.fixture(scope="module")
-    def command_process():
+    def command_process(db_using):
         import os
         if os.environ.get('DJANGO_SETTINGS_MODULE'):
             os.environ.pop('DJANGO_SETTINGS_MODULE')
-        server = subprocess.Popen(cmd, env=os.environ.copy(), cwd=str(cwd or os.getcwd()))
+        env = os.environ.copy()
+        env['UTILMETA_OPERATIONS_DATABASE_ENGINE'] = db_using
+        server = subprocess.Popen(cmd, env=env, cwd=str(cwd or os.getcwd()))
 
         try:
             if port:

tests/server/app/api.py

@@ -87,3 +87,16 @@ def patch(self, data: UserBase[orm.WP], user: User = session_user) -> UserBase:
         data.id = user.pk
         data.save()
         return UserBase.init(data.id)
+
+    from utilmeta.utils import print_time
+
+    @api.get
+    @print_time
+    async def test(self, using='postgresql'):
+        from app.schema import UserSchema
+        from app.models import User
+        return await UserSchema.ainit(
+            User.objects.filter(
+                username='alice',
+            ).using(using),
+        )

tests/server/server.py

@@ -63,7 +63,8 @@ class ServiceEnvironment(Env):
         name=DB_OPS_PATH,
         engine='sqlite3',
     ),
-    secret_names=[Operations.DEFAULT_SECRET_NAMES, 'token']
+    secret_names=[Operations.DEFAULT_SECRET_NAMES, 'token'],
+    # monitor=Operations.Monitor(database_disabled=True)
 ))
 service.use(DatabaseConnections({
     'default': Database(
@@ -72,6 +73,22 @@ class ServiceEnvironment(Env):
         # user=env.DB_USER,
         # password=env.DB_PASSWORD,
         # port=env.DB_PORT
+    ),
+    'postgresql': Database(
+        name='utilmeta_test_db',
+        engine='postgresql',
+        host='127.0.0.1',
+        port=5432,
+        user='test_user',
+        password='test-tmp-password',
+    ),
+    'mysql': Database(
+        name='utilmeta_test_db',
+        engine='mysql',
+        host='127.0.0.1',
+        port=3306,
+        user='test_user',
+        password='test-tmp-password',
     )
 }))
 service.use(CacheConnections({

tests/test_1_orm/test_prepare_data.py

@@ -3,7 +3,7 @@
 setup_service(__name__)
 
 
-def test_prepare_data(service):
+def test_prepare_data(service, db_using):
     # from utilmeta.utils import exceptions as exc
     from django.db.utils import OperationalError, ProgrammingError
     # from server import service
@@ -16,17 +16,17 @@ def test_prepare_data(service):
     from utilmeta.core import orm
 
     try:
-        User.objects.exists()
+        User.objects.using(db_using).exists()
     except (OperationalError, ProgrammingError):
         from django.core.management import execute_from_command_line
-        execute_from_command_line([__name__, 'migrate'])
+        execute_from_command_line([__name__, 'migrate', f'--database={db_using}'])
         # os.system("python -m utilmeta migrate")
 
     # delete all data
-    User.objects.all().delete()
-    Article.objects.all().delete()
-    Comment.objects.all().delete()
-    Follow.objects.all().delete()
+    User.objects.all().using(db_using).delete()
+    Article.objects.all().using(db_using).delete()
+    Comment.objects.all().using(db_using).delete()
+    Follow.objects.all().using(db_using).delete()
 
     # test bulk create
     UserSchema[orm.A].bulk_save(
@@ -58,9 +58,10 @@ def test_prepare_data(service):
                 password="sudo-123",
             ),
         ],
+        using=db_using
     )
 
-    assert User.objects.count() == 5
+    assert User.objects.using(db_using).count() == 5
 
     objs = [
         Follow(id=1, target_id=1, user_id=2),
@@ -70,9 +71,9 @@ def test_prepare_data(service):
         Follow(id=5, target_id=3, user_id=2),
     ]
     for obj in objs:
-        obj.save()
+        obj.save(using=db_using)
 
-    assert Follow.objects.count() == 5
+    assert Follow.objects.using(db_using).count() == 5
 
     ArticleSchema[orm.A].bulk_save(
         [
@@ -119,6 +120,7 @@ def test_prepare_data(service):
                 public=False
             ),
         ],
+        using=db_using
     )
 
     CommentSchema[orm.A].bulk_save([
@@ -127,19 +129,19 @@ def test_prepare_data(service):
         dict(id=8, author_id=2, on_content_id=2, content="lol!"),
         dict(id=9, author_id=4, on_content_id=4, content="wow"),
         dict(id=10, author_id=5, on_content_id=4, content="brilliant~"),
-    ])
+    ], using=db_using)
 
-    assert sorted([val.pk for val in Article.objects.all()]) == [1, 2, 3, 4, 5]
-    assert sorted([val.pk for val in Comment.objects.all()]) == [6, 7, 8, 9, 10]
-    assert BaseContent.objects.filter(public=True).count() == 9
-    assert BaseContent.objects.filter(type="comment").count() == 5
+    assert sorted([val.pk for val in Article.objects.all().using(db_using)]) == [1, 2, 3, 4, 5]
+    assert sorted([val.pk for val in Comment.objects.all().using(db_using)]) == [6, 7, 8, 9, 10]
+    assert BaseContent.objects.filter(public=True).using(db_using).count() == 9
+    assert BaseContent.objects.filter(type="comment").using(db_using).count() == 5
 
-    article_1 = Article.objects.get(id=1)
-    article_2 = Article.objects.get(id=2)
-    article_3 = Article.objects.get(id=3)
-    article_4 = Article.objects.get(id=4)
-    comment_6 = Comment.objects.get(id=6)
-    comment_7 = Comment.objects.get(id=7)
+    article_1 = Article.objects.using(db_using).get(id=1)
+    article_2 = Article.objects.using(db_using).get(id=2)
+    article_3 = Article.objects.using(db_using).get(id=3)
+    article_4 = Article.objects.using(db_using).get(id=4)
+    comment_6 = Comment.objects.using(db_using).get(id=6)
+    comment_7 = Comment.objects.using(db_using).get(id=7)
 
     article_1.liked_bys.set([1, 3, 4])
     article_2.liked_bys.set([5])