add operations compat for django 3.0,3.1,3.2

examples/user_auth/api.py

@@ -0,0 +1,7 @@
+from utilmeta.core import api
+from user.api import UserAPI
+
+
+@api.CORS(allow_origin='*')
+class RootAPI(api.API):
+    user: UserAPI

examples/user_auth/server.py

@@ -2,18 +2,20 @@
 This is a simple one-file project alternative when you setup UtilMeta project
 """
 from utilmeta import UtilMeta
-from utilmeta.core import api
 import django
 
 service = UtilMeta(
     __name__,
     name='user-auth',
     backend=django,
     port=8003,
+    route='/api',
+    api='api.RootAPI'
 )
 
 from utilmeta.core.server.backends.django import DjangoSettings
 from utilmeta.core.orm import DatabaseConnections, Database
+from utilmeta.ops import Operations
 
 service.use(DjangoSettings(
     secret_key='YOUR_SECRET_KEY',
@@ -26,17 +28,14 @@
         engine='sqlite3',
     )
 }))
+service.use(Operations(
+    route='ops',
+    database=Database(
+        name='operations_db',
+        engine='sqlite3',
+    )
+))
 
-service.setup()
-
-from user.api import UserAPI
-
-
-class RootAPI(api.API):
-    user: UserAPI  # new
-
-
-service.mount(RootAPI, route='/api')
 app = service.application()     # used in wsgi/asgi server
 
 if __name__ == '__main__':

utilmeta/core/orm/backends/django/deletion.py

@@ -5,7 +5,7 @@
 from collections import defaultdict
 from itertools import chain
 from django.db.models.deletion import get_candidate_relations_to_delete, \
-    DO_NOTHING, ProtectedError, RestrictedError
+    DO_NOTHING, ProtectedError
 from django.db.models import QuerySet, sql, signals
 from django.db import models
 from django.core.exceptions import EmptyResultSet
@@ -14,10 +14,22 @@
 from functools import reduce
 from operator import attrgetter, or_
 
+try:
+    from django.db.models.deletion import RestrictedError
+except ImportError:
+    class RestrictedError(Exception):
+        pass
+
 
 class AwaitableCollector(Collector):
     connections_cls = DatabaseConnections
 
+    @property
+    def origin_kwargs(self):
+        if django.VERSION >= (4, 0):
+            return dict(origin=self)
+        return {}
+
     @classmethod
     async def delete_single(cls, qs: QuerySet, db: DatabaseConnections.database_cls):
         query = qs.query.clone()
@@ -35,7 +47,11 @@ async def update_batch(cls, model, pk_list, values, db: DatabaseConnections.data
         query = sql.UpdateQuery(model)
         query.add_update_values(values)
         for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE):
-            query.clear_where()
+            if django.VERSION >= (4, 0):
+                query.clear_where()
+            else:
+                from django.db.models.sql.where import WhereNode
+                query.where = WhereNode()
             query.add_filter(
                 "pk__in", pk_list[offset: offset + GET_ITERATOR_CHUNK_SIZE]
             )
@@ -55,7 +71,11 @@ async def delete_batch(cls, model, pk_list, db: DatabaseConnections.database_cls
         num_deleted = 0
         field = query.get_meta().pk
         for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE):
-            query.clear_where()
+            if django.VERSION >= (4, 0):
+                query.clear_where()
+            else:
+                from django.db.models.sql.where import WhereNode
+                query.where = WhereNode()
             query.add_filter(
                 f"{field.attname}__in",
                 pk_list[offset: offset + GET_ITERATOR_CHUNK_SIZE],
@@ -107,7 +127,7 @@ async def async_delete(self):
                         sender=model,
                         instance=obj,
                         using=self.using,
-                        origin=self.origin,
+                        **self.origin_kwargs,
                     )
 
             # fast deletes
@@ -174,7 +194,7 @@ async def async_delete(self):
                             sender=model,
                             instance=obj,
                             using=self.using,
-                            origin=self.origin,
+                            **self.origin_kwargs,
                         )
 
         if django.VERSION < (4, 2):

utilmeta/core/orm/backends/django/query.py

@@ -70,7 +70,13 @@ async def async_pre_sql_setup(self):
 
         # Now we adjust the current query: reset the where clause and get rid
         # of all the tables we don't need (since they're in the sub-select).
-        self.query.clear_where()
+
+        if django.VERSION >= (4, 0):
+            self.query.clear_where()
+        else:
+            from django.db.models.sql.where import WhereNode
+            self.query.where = WhereNode()
+
         if self.query.related_updates or must_pre_select:
             # Either we're using the idents in multiple update queries (so
             # don't want them to change), or the db backend doesn't support
@@ -92,11 +98,19 @@ async def async_pre_sql_setup(self):
             #     for parent, index in related_ids_index:
             #         related_ids[parent].extend(r[index] for r in rows)
 
-            self.query.add_filter("pk__in", idents)
+            filters = ("pk__in", idents)
+            if django.VERSION >= (4, 0):
+                self.query.add_filter(*filters)
+            else:
+                self.query.add_filter(filters)
             self.query.related_ids = related_ids
         else:
             # The fast path. Filters and updates in one query.
-            self.query.add_filter("pk__in", query)
+            filters = ("pk__in", query)
+            if django.VERSION >= (4, 0):
+                self.query.add_filter(*filters)
+            else:
+                self.query.add_filter(filters)
         self.query.reset_refcounts(refcounts_before)
 
     async def async_execute_sql(self, case_update: bool = False):

utilmeta/core/orm/backends/django/queryset.py

@@ -4,7 +4,6 @@
 from django.db.models.query import ValuesListIterable, NamedValuesListIterable, \
     FlatValuesListIterable, ModelIterable
 from django.core import exceptions
-from django.db.models.utils import resolve_callables
 from django.utils.functional import partition
 from utilmeta.utils import awaitable
 from ...databases import DatabaseConnections
@@ -13,7 +12,14 @@
 import django
 from .deletion import AwaitableCollector
 from typing import Optional, Tuple
-from .query import AwaitableQuery, AwaitableSQLUpdateCompiler
+from .query import AwaitableQuery, AwaitableSQLUpdateCompiler, clear_query_ordering
+
+try:
+    from django.db.models.utils import resolve_callables
+except ImportError:
+    def resolve_callables(mapping):
+        for k, v in mapping.items():
+            yield k, v() if callable(v) else v
 
 
 class DummyContent:
@@ -743,9 +749,13 @@ async def adelete(self):
         # Disable non-supported fields.
         del_query.query.select_for_update = False
         del_query.query.select_related = False
-        del_query.query.clear_ordering(force=True)
 
-        collector = self.collector_cls(using=del_query.db, origin=self)
+        clear_query_ordering(del_query.query, True)
+
+        kwargs = dict(using=del_query.db)
+        if django.VERSION >= (4, 0):
+            kwargs.update(origin=self)
+        collector = self.collector_cls(**kwargs)
         if collector.can_fast_delete(del_query):
             await collector.acollect(del_query)
         else:

utilmeta/core/response/backends/django.py

@@ -1,10 +1,13 @@
 from django.http.response import StreamingHttpResponse, HttpResponse, HttpResponseBase, FileResponse
 from typing import Union, TYPE_CHECKING
 from .base import ResponseAdaptor
+import django
 
 if TYPE_CHECKING:
     from utilmeta.core.response import Response
 
+pass_headers = django.VERSION >= (3, 2)
+
 
 class DjangoResponseAdaptor(ResponseAdaptor):
     response: Union[HttpResponse, StreamingHttpResponse]
@@ -24,17 +27,22 @@ def reconstruct(cls, resp: Union['ResponseAdaptor', 'Response']):
         elif not isinstance(resp, Response):
             resp = Response(resp)
 
-        kwargs = dict(
+        kwargs: dict = dict(
             status=resp.status,
             reason=resp.reason,
             content_type=resp.content_type,
             charset=resp.charset,
-            headers=resp.prepare_headers()
         )
+        headers = resp.prepare_headers()
+        if pass_headers:
+            kwargs.update(headers=headers)
         # if resp.file:
         #     response = FileResponse(resp.file, **kwargs)
         # else:
         response = HttpResponse(resp.body, **kwargs)
+        if not pass_headers:
+            for k, v in headers:
+                response[k] = v
         return response
 
     @property
@@ -47,7 +55,9 @@ def reason(self):
 
     @property
     def headers(self):
-        return self.response.headers
+        if django.VERSION >= (3, 2):
+            return self.response.headers
+        return {k: v for k, v in self.response.items()}
 
     @property
     def body(self) -> bytes:

utilmeta/core/server/backends/django/adaptor.py

@@ -30,6 +30,19 @@
 _current_response = contextvars.ContextVar('_django.response')
 
 
+try:
+    from django.utils.decorators import sync_and_async_middleware
+except ImportError:
+    def sync_and_async_middleware(func):
+        """
+        Mark a middleware factory as returning a hybrid middleware supporting both
+        types of request.
+        """
+        func.sync_capable = True
+        func.async_capable = True
+        return func
+
+
 class DebugCookieMiddleware(MiddlewareMixin):
     def process_response(self, request, response: HttpResponseBase):
         origin = request.headers.get(Header.ORIGIN)
@@ -136,7 +149,6 @@ def middleware_func(self):
         if not self.middlewares:
             return None
 
-        from asgiref.sync import iscoroutinefunction
         middlewares = self.middlewares
         request_adaptor_cls = self.request_adaptor_cls
         response_adaptor_cls = self.response_adaptor_cls
@@ -202,12 +214,10 @@ def __call__(self, request):
                 response = self.process_request(request) or self.get_response(request)
                 return self.process_response(response)
 
-        from django.utils.decorators import sync_and_async_middleware
-
         @sync_and_async_middleware
         def utilmeta_middleware(get_response):
             # One-time configuration and initialization goes here.
-            if self.asynchronous and iscoroutinefunction(get_response):
+            if self.asynchronous and inspect.iscoroutinefunction(get_response):
                 async def middleware_func(request):
                     middleware = UtilMetaMiddleware(get_response)
                     # Do something here!

utilmeta/ops/__init__.py

@@ -2,3 +2,59 @@
 __website__ = 'https://ops.utilmeta.com'
 
 from .config import Operations
+
+
+def init_models():
+    import django
+    if django.VERSION < (3, 1):
+        from django.db.models import Field, PositiveIntegerField
+
+        class JSONField(Field):
+            def __init__(self, verbose_name=None, name=None, encoder=None, decoder=None, **kwargs):
+                from utype.utils.encode import JSONEncoder
+                self.encoder = encoder or JSONEncoder
+                self.decoder = decoder
+                super().__init__(verbose_name, name, **kwargs)
+
+            def get_internal_type(self):
+                # act like TextField
+                # return 'JSONField'
+                return 'TextField'
+
+            def db_type(self, connection):
+                return 'text'
+
+            def from_db_value(self, value, expression, connection):
+                if value is not None:
+                    return self.to_python(value)
+                return value
+
+            def to_python(self, value):
+                import json
+                if value is not None:
+                    try:
+                        return json.loads(value)
+                    except (TypeError, ValueError):
+                        return value
+                return value
+
+            def get_prep_value(self, value):
+                import json
+                if value is not None:
+                    return json.dumps(value, cls=self.encoder, ensure_ascii=False)
+                return value
+
+            def value_to_string(self, obj):
+                return self.value_from_object(obj)
+
+            def get_db_prep_value(self, value, connection, prepared=False):
+                import json
+                if value is not None:
+                    return json.dumps(value, cls=self.encoder, ensure_ascii=False)
+                return value
+
+        django.db.models.JSONField = JSONField
+        django.db.models.PositiveBigIntegerField = PositiveIntegerField
+
+
+init_models()

utilmeta/ops/aggregation.py

@@ -106,7 +106,8 @@ def aggregate_logs(service: str,
         total_requests = worker_qs.aggregate(v=models.Sum('requests'))['v'] or requests
         if total_requests:
             avg_time = worker_qs.aggregate(
-                v=models.Sum(models.F('avg_time') * models.F('requests')) / total_requests)['v'] or 0
+                v=models.Sum(models.F('avg_time') * models.F('requests'),
+                             output_field=models.DecimalField()) / total_requests)['v'] or 0
         else:
             avg_time = 0
 

utilmeta/ops/config.py

@@ -1,5 +1,7 @@
 import threading
 
+import django
+
 from utilmeta.conf import Config
 from utilmeta.core.orm.databases.config import Database, DatabaseConnections
 from utype.types import *
@@ -639,3 +641,4 @@ def generator_func(service: 'UtilMeta'):
     #         raise TypeError(f'Invalid application: {app} for django ninja. FastAPI() instance expected')
     #
     #     return generator_func
+