fix bugs on Retry plugin, alter orm queries to compat django 3, patch…

… prepare_data for postgresql

.github/workflows/test.yaml

@@ -90,7 +90,7 @@ jobs:
           pip install flake8 pytest pytest-cov pytest-asyncio
           pip install jwcrypto psutil jwt
           pip install utype
-          pip install databases[aiosqlite] redis aioredis
+          pip install databases[aiosqlite] redis aioredis psycopg2 mysqlclient
           pip install django==${{ matrix.django-version }}
           pip install flask apiflask fastapi sanic[ext] tornado aiohttp uvicorn httpx requests python-multipart
       - name: Install conditional dependencies

tests/conftest.py

@@ -3,6 +3,8 @@
 import time
 import subprocess
 import signal
+
+import django
 import pytest
 import psutil
 from utilmeta import UtilMeta
@@ -45,8 +47,8 @@ def db_using(request):
     return request.param
 
 
-def get_operations_db():
-    engine = os.environ.get('UTILMETA_OPERATIONS_DATABASE_ENGINE') or 'sqlite3'
+def get_operations_db(engine: str = None):
+    engine = engine or os.environ.get('UTILMETA_OPERATIONS_DATABASE_ENGINE') or 'sqlite3'
     from utilmeta.core.orm import Database
     if engine == 'mysql':
         return Database(
@@ -69,6 +71,20 @@ def get_operations_db():
     return Database(engine='sqlite3', name='operations_db')
 
 
+# def patch_time():
+#     if os.environ.get('UTILMETA_OPERATIONS_DATABASE_ENGINE') == 'postgresql' and django.VERSION <= (3, 2):
+#         try:
+#             from utilmeta import service
+#         except ImportError:
+#             pass
+#         else:
+#             from utilmeta.conf import Time
+#             service.use(Time(
+#                 use_tz=False
+#                 # temporary fix problem: database connection isn't set to UTC on lower django version
+#             ))
+
+
 def setup_service(
     name,
     backend: str = None,

tests/server/api.py

@@ -374,9 +374,9 @@ def file(self, content: str):
         return self.response(file=BytesIO(content.encode()))
 
     @api.get
-    @api.Retry(max_retries=3, retry_interval=0.2, max_retries_timeout=.35)
+    @api.Retry(max_retries=3, retry_interval=0.2)
     def retry(self):
-        raise exceptions.BadRequest('retry')
+        raise exceptions.BadRequest(f'retry: {self.request.adaptor.get_context("retry_index")}')
 
     @api.get
     @TestPlugin(num=1)

tests/server/server.py

@@ -14,6 +14,7 @@
 # import django
 # import sanic
 import sys
+import django
 
 try:
     from sanic import Sanic
@@ -52,7 +53,7 @@ class ServiceEnvironment(Env):
 from utilmeta.core.orm import DatabaseConnections, Database
 from utilmeta.core.cache import CacheConnections, Cache
 from utilmeta.ops.config import Operations
-from utilmeta.conf import Preference
+from utilmeta.conf import Preference, Time
 
 service.use(DjangoSettings(
     apps=['app']
@@ -100,6 +101,10 @@ class ServiceEnvironment(Env):
     default_aborted_response_status=500,
     default_timeout_response_status=500
 ))
+service.use(Time(
+    use_tz=django.VERSION > (3, 2)
+    # temporary fix problem: database connection isn't set to UTC on lower django version
+))
 
 # ------ SET BACKEND
 backend = None

tests/test_1_orm/test_prepare_data.py

@@ -1,6 +1,6 @@
-from tests.conftest import setup_service
+from tests.conftest import setup_service, db_using
 #
-setup_service(__name__)
+setup_service(__name__, async_param=False)
 
 
 def test_prepare_data(service, db_using):
@@ -131,6 +131,16 @@ def test_prepare_data(service, db_using):
         dict(id=10, author_id=5, on_content_id=4, content="brilliant~"),
     ], using=db_using)
 
+    # reset sequences for PostgreSQL
+    if db_using == 'postgresql':
+        from django.db import connections
+        for model in [User, BaseContent, Follow]:
+            with connections[db_using].cursor() as cursor:
+                table_name = model._meta.db_table
+                max_id = model.objects.count()
+                sql = f"SELECT setval(pg_get_serial_sequence('{table_name}', 'id'), {max_id});"
+                cursor.execute(sql)
+
     assert sorted([val.pk for val in Article.objects.all().using(db_using)]) == [1, 2, 3, 4, 5]
     assert sorted([val.pk for val in Comment.objects.all().using(db_using)]) == [6, 7, 8, 9, 10]
     assert BaseContent.objects.filter(public=True).using(db_using).count() == 9

tests/test_4_client/test_client.py

@@ -85,8 +85,9 @@ def test_live_server(self, server_thread, sync_request_backend):
 
             #
             tr = client.get_retry()
-            assert tr.status == 500
-            assert 'MaxRetriesTimeoutExceed' in tr.text
+            # assert tr.status == 500
+            # assert 'MaxRetriesTimeoutExceed' in tr.text
+            assert f'retry: 2' in tr.text
 
     def test_live_server_with_mount(self, server_thread, sync_request_backend):
         with APIClient(

tests/test_7_ops/test_ops.py

@@ -1,7 +1,8 @@
 import pytest
-from tests.conftest import make_cmd_process
+from tests.conftest import make_cmd_process, db_using
 from utilmeta.core import cli
 from utilmeta.ops.client import OperationsClient
+from utilmeta.core.api.plugins.retry import RetryPlugin
 # test import client here, client should not depend on ops models
 import utilmeta
 from utilmeta.ops import __spec_version__
@@ -47,11 +48,18 @@
     port=9090
 )
 
+retry = RetryPlugin(
+    max_retries=3, max_retries_timeout=15, retry_interval=1
+)
+
 
 class TestOperations:
     if django.VERSION >= (4, 0):
         def test_django_operations(self, django_wsgi_process):
-            with OperationsClient(base_url='http://127.0.0.1:9091/ops') as client:
+            with OperationsClient(
+                base_url='http://127.0.0.1:9091/ops',
+                plugins=[retry]
+            ) as client:
                 info = client.get_info()
                 assert info.result.utilmeta == __spec_version__
 
@@ -90,7 +98,10 @@ def test_django_operations(self, django_wsgi_process):
                 assert isinstance(data, dict) and data.get('result') == 3
 
         def test_django_asgi_operations(self, django_asgi_process):
-            with OperationsClient(base_url='http://127.0.0.1:9100/ops') as client:
+            with OperationsClient(
+                base_url='http://127.0.0.1:9100/ops',
+                plugins=[retry]
+            ) as client:
                 info = client.get_info()
                 assert info.result.utilmeta == __spec_version__
 
@@ -123,7 +134,7 @@ def test_django_asgi_operations(self, django_asgi_process):
     def test_fastapi_operations(self, fastapi_process):
         with OperationsClient(base_url='http://127.0.0.1:9092/api/v1/ops', base_headers={
             'cache-control': 'no-cache'
-        }) as client:
+        }, plugins=[retry]) as client:
             info = client.get_info()
             assert info.result.utilmeta == __spec_version__
 
@@ -159,7 +170,7 @@ def test_fastapi_operations(self, fastapi_process):
     def test_flask_operations(self, flask_process):
         with OperationsClient(base_url='http://127.0.0.1:9093/ops', base_headers={
             'cache-control': 'no-cache'
-        }) as client:
+        }, plugins=[retry]) as client:
             info = client.get_info()
             assert info.result.utilmeta == __spec_version__
 
@@ -192,7 +203,7 @@ def test_flask_operations(self, flask_process):
     def test_sanic_operations(self, sanic_process):
         with OperationsClient(base_url='http://127.0.0.1:9094/ops', base_headers={
             'cache-control': 'no-cache'
-        }) as client:
+        }, plugins=[retry]) as client:
             info = client.get_info()
             assert info.result.utilmeta == __spec_version__
 
@@ -224,7 +235,7 @@ def test_sanic_operations(self, sanic_process):
             assert 'Hello' in str(hello.data)
 
     def test_tornado_operations(self, tornado_process):
-        with OperationsClient(base_url='http://127.0.0.1:9095/v1/ops') as client:
+        with OperationsClient(base_url='http://127.0.0.1:9095/v1/ops', plugins=[retry]) as client:
             info = client.get_info()
             assert info.result.utilmeta == __spec_version__
 
@@ -246,7 +257,7 @@ def test_tornado_operations(self, tornado_process):
             assert '2.6.0' <= inst.utilmeta_version
 
     def test_utilmeta_operations(self, utilmeta_process):
-        with OperationsClient(base_url='http://127.0.0.1:9090/api/ops') as client:
+        with OperationsClient(base_url='http://127.0.0.1:9090/api/ops', plugins=[retry]) as client:
             info = client.get_info()
             assert info.result.utilmeta == __spec_version__
 

utilmeta/core/api/base.py

@@ -475,6 +475,10 @@ def __mount__(
         if isinstance(handler, APIRoute):
             cls._routes.append(handler)
             return
+        if any([r.handler == handler and r.route == route for r in cls._routes]):
+            # same route and handler, return
+            return
+
         api_route = cls._route_cls(
             handler=handler,
             route=route,

utilmeta/core/api/plugins/retry.py

@@ -109,15 +109,17 @@ def process_request(self, request: Request):
     def handle_max_retries_timeout(self, request: Request, set_timeout: bool = False):
         if not self.max_retries_timeout:
             return
+        current_retry = request.adaptor.get_context("retry_index") or 0
         start_time = request.time
         current_time = time_now()
         delta = (current_time - start_time).total_seconds() - self.max_retries_timeout
-        if delta <= 0:
-            # max retries time exceeded
-            raise self.max_retries_timeout_error_cls(
-                f"{self.__class__}: max_retries_timeout exceed for {abs(delta)} seconds",
-                max_retries_timeout=self.max_retries_timeout,
-            )
+        if current_retry > 0:
+            if delta > 0:
+                # max retries time exceeded
+                raise self.max_retries_timeout_error_cls(
+                    f"{self.__class__}: max_retries_timeout exceed for {abs(delta)} seconds {start_time} {current_time}",
+                    max_retries_timeout=self.max_retries_timeout,
+                )
 
         # reset request timeout
         if set_timeout:
@@ -143,14 +145,15 @@ def handle_max_retries_timeout(self, request: Request, set_timeout: bool = False
                 request.adaptor.update_context(timeout=timeout)
 
             to = request.adaptor.get_context("timeout")
-            if not to or to > delta:
-                request.adaptor.update_context(timeout=delta)
+            remaining_timeout = abs(delta)
+            if not to or to > remaining_timeout:
+                request.adaptor.update_context(timeout=remaining_timeout)
 
     def process_response(self, response: Response):
         request = response.request
         if not request:
             return response
-        current_retry = request.get_context("retry_index") or 0
+        current_retry = request.adaptor.get_context("retry_index") or 0
         if current_retry + 1 >= self.max_retries:
             # cannot retry
             return response
@@ -166,7 +169,7 @@ async def process_response(self, response: Response):
         request = response.request
         if not request:
             return response
-        current_retry = request.get_context("retry_index") or 0
+        current_retry = request.adaptor.get_context("retry_index") or 0
         if current_retry + 1 >= self.max_retries:
             # cannot retry
             return response

utilmeta/core/orm/backends/django/generator.py

@@ -1,3 +1,5 @@
+import django
+
 from . import expressions as exp
 from ..base import ModelFieldAdaptor
 from utilmeta.core.orm.fields.filter import ParserFilter
@@ -124,7 +126,9 @@ def _add_annotate(
             raise TypeError(f"Invalid expression: {expression}")
         if distinct_count and isinstance(expression, exp.Count):
             expression.distinct = True
-        if isinstance(expression, exp.Sum):
+        if isinstance(expression, exp.Sum) and django.VERSION >= (4, 0):
+            # apply to django 4+ only
+            # or will cause ProgrammingError
             expression = exp.Subquery(
                 models.QuerySet(model=self.model.model)
                 .filter(pk=exp.OuterRef("pk"))