diff --git a/docs/api/ming.odm.rst b/docs/api/ming.odm.rst index ddc1162..59fd955 100644 --- a/docs/api/ming.odm.rst +++ b/docs/api/ming.odm.rst @@ -4,6 +4,8 @@ .. automodule:: ming.odm.declarative :members: + :show-inheritance: + :inherited-members: .. automodule:: ming.odm.base :members: diff --git a/docs/encryption.rst b/docs/encryption.rst index d78fef1..ea6dee6 100644 --- a/docs/encryption.rst +++ b/docs/encryption.rst @@ -13,12 +13,15 @@ This section describes how Ming can be used to automatically encrypt and decrypt .. _Client-Side Field Level Encryption (CSFLE): https://pymongo.readthedocs.io/en/stable/examples/encryption.html#client-side-field-level-encryption -Declarative Field-Level Encryption +Encryption at the Foundation Level ================================== -When declaratively working with models by subclassing the :class:`ming.Document` in the :ref:`ming_baselevel` this is accomplished by pairing a :class:`~ming.encryption.DecryptedField` with a :class:`~ming.metadata.Field`. +When declaratively working with models by subclassing :class:`~ming.declarative.Document` in the :ref:`ming_baselevel`, you can add field level encryption by pairing a :class:`~ming.encryption.DecryptedField` with a :class:`~ming.metadata.Field`. -A simple example might look like the following.:: + +A simple example might look like the following. + +.. code-block:: python class UserEmail(Document): class __mongometa__: @@ -30,10 +33,57 @@ A simple example might look like the following.:: email = DecryptedField(str, 'email_encrypted') -Breaking it Down -======================== +Breaking down DecryptedField +---------------------------------- + +This approach requires that you follow a few conventions: + +#. The field storing the encrypted data should be configured in the following way: + + * It should be a :class:`~ming.metadata.Field`. + * The Field should be of type :class:`~ming.schema.Binary`. + * The Field's name should end with `_encrypted`. + +#. Next to this should be a corresponding :class:`~ming.encryption.DecryptedField` that will decrypt the data. + + * Its first argument should be the type that you expect the decrypted data to be (`str`, `int`, etc.). + * The second argument should be the name of the encrypted field (e.g. `email_encrypted`). + * The DecryptedField's name should be the same as the encrypted :class:`~ming.metadata.Field`, but without the `_encrypted` suffix (e.g. `email`). + + +Encryption at the Declarative Level +======================================== + +Similarly when working with the higher level of abstraction offered by :class:`~ming.odm.declarative.MappedClass`es, you can add field level encryption by pairing a :class:`~ming.odm.declarative.DecryptedProperty` with a :class:`~ming.odm.property.FieldProperty` + + +A simple example might look like the following. + +.. code-block:: python + + class UserEmail(MappedClass): + class __mongometa__: + session = session + name = 'user_emails' + _id = FieldProperty(schema.ObjectId) + + email_encrypted = FieldProperty(S.Binary, if_missing=None) + email = DecryptedProperty(str, 'email_encrypted') + + +Breaking down DecryptedProperty +---------------------------------- + +Similarly to the foundation level, this approach requires that you follow a few conventions: + +#. The field storing the encrypted data should be configured in the following way: -This approach requires that you follow a few conventions in order to function correctly. + * It should be a :class:`~ming.odm.property.FieldProperty`. + * The FieldProperty should be of type :class:`~ming.schema.Binary`. + * The FieldProperty's name should end with `_encrypted`. -.. 1. Fields encrypted data must be named with the suffix `_encrypted`. +#. Next to this should be a :class:`~ming.odm.declarative.DecryptedProperty` that will decrypt the data. + * Its first argument should be the type that you expect the decrypted data to be (`str`, `int`, etc.). + * The second argument should be the name of the encrypted field (e.g. `email_encrypted`). + * The DecryptedProperty's name should be the same as the encrypted :class:`~ming.odm.declarative.DecryptedProperty`, but without the `_encrypted` suffix (e.g. `email`). diff --git a/ming/datastore.py b/ming/datastore.py index 047a640..29fe2c2 100644 --- a/ming/datastore.py +++ b/ming/datastore.py @@ -237,7 +237,7 @@ def encr(self, s: str | None, _first_attempt=True, provider='local') -> bytes | if s is None: return None try: - key_alt_name = self.encryption.get_key_alt_name(provider) + key_alt_name = self.encryption._get_key_alt_name(provider) return self.encryptor.encrypt(s, Algorithm.AEAD_AES_256_CBC_HMAC_SHA_512_Deterministic, key_alt_name=key_alt_name) except (EncryptionError, MongoCryptError) as e: diff --git a/ming/encryption.py b/ming/encryption.py index 7d26ca4..990c8dc 100644 --- a/ming/encryption.py +++ b/ming/encryption.py @@ -1,12 +1,10 @@ from __future__ import annotations -from functools import lru_cache from typing import TYPE_CHECKING, TypeVar, Generic from ming.utils import classproperty if TYPE_CHECKING: - from pymongo import MongoClient import ming.datastore @@ -17,8 +15,9 @@ class MingEncryptionError(Exception): class EncryptionConfig: """ A class to hold the encryption configuration for a ming datastore. - :param encryption_config: a dictionary that closely resembles various features of the MongoDB - encryption that we support. + + :param config: a dictionary that closely resembles various features of the MongoDB + encryption that we support. """ def __init__(self, config: dict): @@ -69,7 +68,7 @@ def provider_options(self) -> dict: """ return self._encryption_config.get('provider_options') - def get_key_alt_name(self, provider='local') -> str: + def _get_key_alt_name(self, provider='local') -> str: return self.provider_options.get(provider)['key_alt_names'][0] @property @@ -118,9 +117,12 @@ def __set__(self, instance: EncryptedMixin, value: T): class EncryptedMixin: - """A mixin intended to be used with ming.schema.Document classes to provide encryption. + """A mixin intended to be used with :class:`~ming.declarative.Document` + or :class:`~ming.odm.declarative.MappedClass` to provide encryption. All configuration is handled by an instance of a :class:`ming.encryption.EncryptionConfig` - that is passed to the :class:`ming.datastore.DataStore` instance that the Document is bound to. + that is passed to the :class:`ming.datastore.DataStore` instance that the Document/MappedClass is bound to. + + Generally, don't use this directly, but instead call the methods on the Document/MappedClass you're working with. """ @classproperty @@ -191,7 +193,9 @@ def encrypt_some_fields(cls, data: dict) -> dict: return encrypted_data def decrypt_some_fields(self) -> dict: - # useful for json, removes encrypted fields and uses decrypted forms + """ + Returns a `dict` with raw data. Removes encrypted fields and replaces them with decrypted data. Useful for json. + """ decrypted_data = dict() for k in self._field_names: if k.endswith('_encrypted'): diff --git a/ming/odm/declarative.py b/ming/odm/declarative.py index 28c3e35..e10d08f 100644 --- a/ming/odm/declarative.py +++ b/ming/odm/declarative.py @@ -1,9 +1,7 @@ from ming.metadata import collection, Index -from ming.datastore import DataStore from ming.encryption import EncryptedMixin -from ming.utils import classproperty -from .mapper import mapper, Mapper -from .property import ORMProperty +from ming.odm.mapper import mapper +from ming.odm.property import ORMProperty class _MappedClassMeta(type): diff --git a/ming/odm/mapper.py b/ming/odm/mapper.py index d505232..38fb332 100644 --- a/ming/odm/mapper.py +++ b/ming/odm/mapper.py @@ -8,7 +8,7 @@ from ming.session import Session from ming.utils import wordwrap -from .base import ObjectState, ObjectState, state, _with_hooks +from .base import ObjectState, state, _with_hooks from .property import FieldProperty if typing.TYPE_CHECKING: @@ -439,9 +439,11 @@ def __get__(self, self_, cls=None): return self.saving_init(self_) else: return self.nonsaving_init(self_) + + __call__ = __get__ @classmethod - def decorate(cls, mapped_class, mapper): + def decorate(cls, mapped_class: type[MappedClass], mapper: Mapper): old_init = mapped_class.__init__ if isinstance(old_init, cls): mapped_class.__init__ = cls(mapper, old_init.func) diff --git a/ming/odm/mapper.pyi b/ming/odm/mapper.pyi index 02f593f..43ce273 100644 --- a/ming/odm/mapper.pyi +++ b/ming/odm/mapper.pyi @@ -1,24 +1,87 @@ -from typing import type_check_only, TypeVar, Generic, Optional, Union, Any, Dict, overload +from typing import Generator, List, type_check_only, TypeVar, Generic, Optional, Union, Any, overload, Type from bson import ObjectId from pymongo.command_cursor import CommandCursor +import pymongo.results -from ming.base import Cursor -M = TypeVar('M') +from ming import Document, Cursor, Session +from ming.base import Object +from ming.odm import MappedClass, FieldProperty, ODMSession +from ming.odm.base import ObjectState + +TMappedClass = TypeVar('M', bound=MappedClass) +TDocument = TypeVar('D', bound=Document) + +def mapper(cls: Type[TMappedClass], collection: TDocument = None, session: Session = None, **kwargs) -> Mapper: ... + +class Mapper(Generic[TMappedClass]): + + properties: List[FieldProperty] + property_index: dict[str, FieldProperty] + collection: Type[TDocument] + mapped_class: Type[TMappedClass] + session: Session + extensions: list + options: Object + + def __init__(self, mapped_class: Type[TMappedClass], collection: Type[Document], session: Session, **kwargs): ... + + @classmethod + def replace_session(cls, session) -> None: ... + + def insert(self, obj: MappedClass, state: ObjectState, session: ODMSession, **kwargs) -> pymongo.results.InsertOneResult: ... + + def update(self, obj: MappedClass, state: ObjectState, session: ODMSession, **kwargs) -> ObjectId: ... + + def delete(self, obj: MappedClass, state: ObjectState, session: ODMSession, **kwargs) -> pymongo.results.DeleteResult: ... + + def remove(self, session: ODMSession, *args, **kwargs) -> pymongo.results.DeleteResult: ... + + def create(self, doc, options, remake=True) -> TMappedClass: ... + + def base_mappers(self) -> Generator[Mapper]: ... + + def all_properties(self) -> Generator[FieldProperty]: ... + + @classmethod + def by_collection(cls, collection_class: Type[TDocument]) -> Mapper[TMappedClass]: ... + + @classmethod + def by_class(cls, mapped_class: Type[TMappedClass]) -> Mapper[TMappedClass]: ... + + @classmethod + def by_classname(cls, name: str) -> Mapper[TMappedClass]: ... + + @classmethod + def all_mappers(cls) -> List[Mapper[TMappedClass]]: ... + + @classmethod + def compile_all(cls): ... + + @classmethod + def clear_all(cls): ... + + @classmethod + def ensure_all_indexes(cls): ... + + def compile(self): ... + + def update_partial(self, session: ODMSession, *args, **kwargs) -> pymongo.results.UpdateResult: ... + MongoFilter = dict ChangeResult = dict -class _ClassQuery(Generic[M]): +class _ClassQuery(Generic[TMappedClass]): # proxies most of these from Session - def get(self, _id: Union[ObjectId|Any] = None, **kwargs) -> Optional[M]: ... - def find(self, filter: MongoFilter = None, *args, **kwargs) -> Cursor[M]: ... - def find_by(self, filter: MongoFilter = None, *args, **kwargs) -> Cursor[M]: ... + def get(self, _id: Union[ObjectId|Any] = None, **kwargs) -> Optional[TMappedClass]: ... + def find(self, filter: MongoFilter = None, *args, **kwargs) -> Cursor[TMappedClass]: ... + def find_by(self, filter: MongoFilter = None, *args, **kwargs) -> Cursor[TMappedClass]: ... def remove(self, spec_or_id: Union[MongoFilter, ObjectId] = None, **kwargs) -> ChangeResult: ... def count(self) -> int: ... - def find_one_and_update(self, **kwargs) -> M: ... - def find_one_and_replace(self, **kwargs) -> M: ... - def find_one_and_delete(self, **kwargs) -> M: ... + def find_one_and_update(self, **kwargs) -> TMappedClass: ... + def find_one_and_replace(self, **kwargs) -> TMappedClass: ... + def find_one_and_delete(self, **kwargs) -> TMappedClass: ... def update_partial(self, filter: MongoFilter, fields: dict, **kwargs) -> ChangeResult: ... def aggregate(self, pipeline: list, **kwargs) -> CommandCursor: ... def distinct(self, key: str, filter: MongoFilter | None = None, **kwargs) -> list: ... @@ -30,7 +93,7 @@ class _InstQuery(object): def delete(self) -> ChangeResult: ... @type_check_only -class Query(_ClassQuery[M], _InstQuery): +class Query(_ClassQuery[TMappedClass], _InstQuery): @overload # from _ClassQuery def update(self, spec: MongoFilter, fields: dict, **kwargs) -> ChangeResult: ... diff --git a/ming/tests/test_datastore.py b/ming/tests/test_datastore.py index 4fe2407..60a6fee 100644 --- a/ming/tests/test_datastore.py +++ b/ming/tests/test_datastore.py @@ -1,8 +1,7 @@ -import os import sys from unittest import TestCase, main -from unittest.mock import patch, Mock +from unittest.mock import patch from pymongo.errors import ConnectionFailure import ming @@ -11,7 +10,6 @@ from ming import create_datastore, create_engine from ming.exc import MingConfigError from ming.datastore import Engine -from ming.tests import EncryptionConfigHelper, make_encryption_key class DummyConnection: