fixup! fixup! fixup! fixup! fixup! add support for mongodb Client Sid…

…e Field Level Encryption (CSFLE)

docs/api/ming.odm.rst

@@ -4,6 +4,8 @@
 
 .. automodule:: ming.odm.declarative
  :members:
+ :show-inheritance:
+ :inherited-members:
 
 .. automodule:: ming.odm.base
  :members:

docs/encryption.rst

@@ -13,12 +13,15 @@ This section describes how Ming can be used to automatically encrypt and decrypt
 .. _Client-Side Field Level Encryption (CSFLE): https://pymongo.readthedocs.io/en/stable/examples/encryption.html#client-side-field-level-encryption
 
 
-Declarative Field-Level Encryption
+Encryption at the Foundation Level
 ==================================
 
-When declaratively working with models by subclassing the :class:`ming.Document` in the :ref:`ming_baselevel` this is accomplished by pairing a :class:`~ming.encryption.DecryptedField` with a :class:`~ming.metadata.Field`.
+When declaratively working with models by subclassing :class:`~ming.declarative.Document` in the :ref:`ming_baselevel`, you can add field level encryption by pairing a :class:`~ming.encryption.DecryptedField` with a :class:`~ming.metadata.Field`.
 
-A simple example might look like the following.::
+
+A simple example might look like the following.
+
+.. code-block:: python
 
  class UserEmail(Document):
  class __mongometa__:
@@ -30,10 +33,57 @@ A simple example might look like the following.::
  email = DecryptedField(str, 'email_encrypted')
 
 
-Breaking it Down
-========================
+Breaking down DecryptedField
+----------------------------------
+
+This approach requires that you follow a few conventions:
+
+#. The field storing the encrypted data should be configured in the following way:
+
+ * It should be a :class:`~ming.metadata.Field`.
+ * The Field should be of type :class:`~ming.schema.Binary`.
+ * The Field's name should end with `_encrypted`.
+
+#. Next to this should be a corresponding :class:`~ming.encryption.DecryptedField` that will decrypt the data.
+
+ * Its first argument should be the type that you expect the decrypted data to be (`str`, `int`, etc.).
+ * The second argument should be the name of the encrypted field (e.g. `email_encrypted`).
+ * The DecryptedField's name should be the same as the encrypted :class:`~ming.metadata.Field`, but without the `_encrypted` suffix (e.g. `email`).
+
+
+Encryption at the Declarative Level
+========================================
+
+Similarly when working with the higher level of abstraction offered by :class:`~ming.odm.declarative.MappedClass`es, you can add field level encryption by pairing a :class:`~ming.odm.declarative.DecryptedProperty` with a :class:`~ming.odm.property.FieldProperty`
+
+
+A simple example might look like the following.
+
+.. code-block:: python
+ 
+ class UserEmail(MappedClass):
+ class __mongometa__:
+ session = session
+ name = 'user_emails'
+ _id = FieldProperty(schema.ObjectId)
+ 
+ email_encrypted = FieldProperty(S.Binary, if_missing=None)
+ email = DecryptedProperty(str, 'email_encrypted')
+
+
+Breaking down DecryptedProperty
+----------------------------------
+
+Similarly to the foundation level, this approach requires that you follow a few conventions:
+
+#. The field storing the encrypted data should be configured in the following way:
 
-This approach requires that you follow a few conventions in order to function correctly.
+ * It should be a :class:`~ming.odm.property.FieldProperty`.
+ * The FieldProperty should be of type :class:`~ming.schema.Binary`.
+ * The FieldProperty's name should end with `_encrypted`.
 
-.. 1. Fields encrypted data must be named with the suffix `_encrypted`.
+#. Next to this should be a :class:`~ming.odm.declarative.DecryptedProperty` that will decrypt the data.
 
+ * Its first argument should be the type that you expect the decrypted data to be (`str`, `int`, etc.).
+ * The second argument should be the name of the encrypted field (e.g. `email_encrypted`).
+ * The DecryptedProperty's name should be the same as the encrypted :class:`~ming.odm.declarative.DecryptedProperty`, but without the `_encrypted` suffix (e.g. `email`).

ming/datastore.py

@@ -237,7 +237,7 @@ def encr(self, s: str | None, _first_attempt=True, provider='local') -> bytes |
  if s is None:
  return None
  try:
- key_alt_name = self.encryption.get_key_alt_name(provider)
+ key_alt_name = self.encryption._get_key_alt_name(provider)
  return self.encryptor.encrypt(s, Algorithm.AEAD_AES_256_CBC_HMAC_SHA_512_Deterministic,
  key_alt_name=key_alt_name)
  except (EncryptionError, MongoCryptError) as e:

ming/encryption.py

@@ -1,12 +1,10 @@
 from __future__ import annotations
 
-from functools import lru_cache
 from typing import TYPE_CHECKING, TypeVar, Generic
 
 from ming.utils import classproperty
 
 if TYPE_CHECKING:
- from pymongo import MongoClient
  import ming.datastore
 
 
@@ -17,8 +15,9 @@ class MingEncryptionError(Exception):
 class EncryptionConfig:
  """
  A class to hold the encryption configuration for a ming datastore.
- :param encryption_config: a dictionary that closely resembles various features of the MongoDB
- encryption that we support.
+
+ :param config: a dictionary that closely resembles various features of the MongoDB
+ encryption that we support.
  """
 
  def __init__(self, config: dict):
@@ -69,7 +68,7 @@ def provider_options(self) -> dict:
  """
  return self._encryption_config.get('provider_options')
 
- def get_key_alt_name(self, provider='local') -> str:
+ def _get_key_alt_name(self, provider='local') -> str:
  return self.provider_options.get(provider)['key_alt_names'][0]
 
  @property
@@ -118,9 +117,12 @@ def __set__(self, instance: EncryptedMixin, value: T):
 
 
 class EncryptedMixin:
- """A mixin intended to be used with ming.schema.Document classes to provide encryption.
+ """A mixin intended to be used with :class:`~ming.declarative.Document`
+ or :class:`~ming.odm.declarative.MappedClass` to provide encryption.
  All configuration is handled by an instance of a :class:`ming.encryption.EncryptionConfig`
- that is passed to the :class:`ming.datastore.DataStore` instance that the Document is bound to.
+ that is passed to the :class:`ming.datastore.DataStore` instance that the Document/MappedClass is bound to.
+
+ Generally, don't use this directly, but instead call the methods on the Document/MappedClass you're working with.
  """
 
  @classproperty
@@ -191,7 +193,9 @@ def encrypt_some_fields(cls, data: dict) -> dict:
  return encrypted_data
 
  def decrypt_some_fields(self) -> dict:
- # useful for json, removes encrypted fields and uses decrypted forms
+ """
+ Returns a `dict` with raw data. Removes encrypted fields and replaces them with decrypted data. Useful for json.
+ """
  decrypted_data = dict()
  for k in self._field_names:
  if k.endswith('_encrypted'):

ming/odm/declarative.py

@@ -1,9 +1,7 @@
 from ming.metadata import collection, Index
-from ming.datastore import DataStore
 from ming.encryption import EncryptedMixin
-from ming.utils import classproperty
-from .mapper import mapper, Mapper
-from .property import ORMProperty
+from ming.odm.mapper import mapper
+from ming.odm.property import ORMProperty
 
 
 class _MappedClassMeta(type):

ming/odm/mapper.py

@@ -8,7 +8,7 @@
 from ming.session import Session
 from ming.utils import wordwrap
 
-from .base import ObjectState, ObjectState, state, _with_hooks
+from .base import ObjectState, state, _with_hooks
 from .property import FieldProperty
 
 if typing.TYPE_CHECKING:
@@ -439,9 +439,11 @@ def __get__(self, self_, cls=None):
  return self.saving_init(self_)
  else:
  return self.nonsaving_init(self_)
+
+ __call__ = __get__
 
  @classmethod
- def decorate(cls, mapped_class, mapper):
+ def decorate(cls, mapped_class: type[MappedClass], mapper: Mapper):
  old_init = mapped_class.__init__
  if isinstance(old_init, cls):
  mapped_class.__init__ = cls(mapper, old_init.func)

ming/odm/mapper.pyi

@@ -1,24 +1,87 @@
-from typing import type_check_only, TypeVar, Generic, Optional, Union, Any, Dict, overload
+from typing import Generator, List, type_check_only, TypeVar, Generic, Optional, Union, Any, overload, Type
 
 from bson import ObjectId
 from pymongo.command_cursor import CommandCursor
+import pymongo.results
 
-from ming.base import Cursor
 
-M = TypeVar('M')
+from ming import Document, Cursor, Session
+from ming.base import Object
+from ming.odm import MappedClass, FieldProperty, ODMSession
+from ming.odm.base import ObjectState
+
+TMappedClass = TypeVar('M', bound=MappedClass)
+TDocument = TypeVar('D', bound=Document)
+
+def mapper(cls: Type[TMappedClass], collection: TDocument = None, session: Session = None, **kwargs) -> Mapper: ...
+
+class Mapper(Generic[TMappedClass]):
+
+ properties: List[FieldProperty]
+ property_index: dict[str, FieldProperty]
+ collection: Type[TDocument]
+ mapped_class: Type[TMappedClass]
+ session: Session
+ extensions: list
+ options: Object
+
+ def __init__(self, mapped_class: Type[TMappedClass], collection: Type[Document], session: Session, **kwargs): ...
+
+ @classmethod
+ def replace_session(cls, session) -> None: ...
+
+ def insert(self, obj: MappedClass, state: ObjectState, session: ODMSession, **kwargs) -> pymongo.results.InsertOneResult: ...
+
+ def update(self, obj: MappedClass, state: ObjectState, session: ODMSession, **kwargs) -> ObjectId: ...
+
+ def delete(self, obj: MappedClass, state: ObjectState, session: ODMSession, **kwargs) -> pymongo.results.DeleteResult: ...
+
+ def remove(self, session: ODMSession, *args, **kwargs) -> pymongo.results.DeleteResult: ...
+
+ def create(self, doc, options, remake=True) -> TMappedClass: ...
+
+ def base_mappers(self) -> Generator[Mapper]: ...
+
+ def all_properties(self) -> Generator[FieldProperty]: ...
+
+ @classmethod
+ def by_collection(cls, collection_class: Type[TDocument]) -> Mapper[TMappedClass]: ...
+
+ @classmethod
+ def by_class(cls, mapped_class: Type[TMappedClass]) -> Mapper[TMappedClass]: ...
+
+ @classmethod
+ def by_classname(cls, name: str) -> Mapper[TMappedClass]: ...
+
+ @classmethod
+ def all_mappers(cls) -> List[Mapper[TMappedClass]]: ...
+
+ @classmethod
+ def compile_all(cls): ...
+
+ @classmethod
+ def clear_all(cls): ...
+
+ @classmethod
+ def ensure_all_indexes(cls): ...
+
+ def compile(self): ...
+
+ def update_partial(self, session: ODMSession, *args, **kwargs) -> pymongo.results.UpdateResult: ...
+
 
 MongoFilter = dict
 ChangeResult = dict
-class _ClassQuery(Generic[M]):
+class _ClassQuery(Generic[TMappedClass]):
  # proxies most of these from Session
- def get(self, _id: Union[ObjectId|Any] = None, **kwargs) -> Optional[M]: ...
- def find(self, filter: MongoFilter = None, *args, **kwargs) -> Cursor[M]: ...
- def find_by(self, filter: MongoFilter = None, *args, **kwargs) -> Cursor[M]: ...
+ def get(self, _id: Union[ObjectId|Any] = None, **kwargs) -> Optional[TMappedClass]: ...
+ def find(self, filter: MongoFilter = None, *args, **kwargs) -> Cursor[TMappedClass]: ...
+ def find_by(self, filter: MongoFilter = None, *args, **kwargs) -> Cursor[TMappedClass]: ...
  def remove(self, spec_or_id: Union[MongoFilter, ObjectId] = None, **kwargs) -> ChangeResult: ...
  def count(self) -> int: ...
- def find_one_and_update(self, **kwargs) -> M: ...
- def find_one_and_replace(self, **kwargs) -> M: ...
- def find_one_and_delete(self, **kwargs) -> M: ...
+ def find_one_and_update(self, **kwargs) -> TMappedClass: ...
+ def find_one_and_replace(self, **kwargs) -> TMappedClass: ...
+ def find_one_and_delete(self, **kwargs) -> TMappedClass: ...
  def update_partial(self, filter: MongoFilter, fields: dict, **kwargs) -> ChangeResult: ...
  def aggregate(self, pipeline: list, **kwargs) -> CommandCursor: ...
  def distinct(self, key: str, filter: MongoFilter | None = None, **kwargs) -> list: ...
@@ -30,7 +93,7 @@ class _InstQuery(object):
  def delete(self) -> ChangeResult: ...
 
 @type_check_only
-class Query(_ClassQuery[M], _InstQuery):
+class Query(_ClassQuery[TMappedClass], _InstQuery):
 
  @overload # from _ClassQuery
  def update(self, spec: MongoFilter, fields: dict, **kwargs) -> ChangeResult: ...

ming/tests/test_datastore.py

@@ -1,8 +1,7 @@
-import os
 import sys
 from unittest import TestCase, main
 
-from unittest.mock import patch, Mock
+from unittest.mock import patch
 from pymongo.errors import ConnectionFailure
 
 import ming
@@ -11,7 +10,6 @@
 from ming import create_datastore, create_engine
 from ming.exc import MingConfigError
 from ming.datastore import Engine
-from ming.tests import EncryptionConfigHelper, make_encryption_key
 
 
 class DummyConnection: