Add Support For MongoDB Client Side Field Level Encryption (CSFLE)

docs/api/ming.encryption.rst

@@ -0,0 +1,7 @@
+:mod:`ming.encryption` module
+================================
+
+
+.. automodule:: ming.encryption
+ :members:
+ :private-members:

docs/baselevel.rst

@@ -17,7 +17,7 @@ you want.
 While this dynamic behavior is handy in a rapid development environment where you
 might delete and re-create the database many times a day, it starts to be a
 problem when you *need* to make guarantees of the type of data in a collection
-(because you code depends on it). The goal of Ming is to allow you to specify
+(because your code depends on it). The goal of Ming is to allow you to specify
 the schema for your data in Python code and then develop in confidence, knowing
 the format of data you get from a query.
 

docs/encryption.rst

@@ -0,0 +1,39 @@
+:tocdepth: 3
+
+.. _odm-encryption:
+
+============================
+Encrypting Sensitive Data
+============================
+
+This section describes how Ming can be used to automatically encrypt and decrypt your document's fields. This is accomplished by leveraging MongoDB's `Client-Side Field Level Encryption (CSFLE)`_ feature.
+
+
+
+.. _Client-Side Field Level Encryption (CSFLE): https://pymongo.readthedocs.io/en/stable/examples/encryption.html#client-side-field-level-encryption
+
+
+Declarative Field-Level Encryption
+==================================
+
+When declaratively working with models by subclassing the :class:`ming.Document` in the :ref:`ming_baselevel` this is accomplished by pairing a :class:`~ming.encryption.DecryptedField` with a :class:`~ming.metadata.Field`.
+
+A simple example might look like the following.::
+
+ class UserEmail(Document):
+ class __mongometa__:
+ session = session
+ name = 'user_emails'
+ _id = Field(schema.ObjectId)
+
+ email_encrypted = Field(S.Binary, if_missing=None)
+ email = DecryptedField(str, 'email_encrypted')
+
+
+Breaking it Down
+========================
+
+This approach requires that you follow a few conventions in order to function correctly.
+
+.. 1. Fields encrypted data must be named with the suffix `_encrypted`.
+

docs/index.rst

@@ -101,6 +101,7 @@ Documentation Content
  polymorphism
  custom_properties
  baselevel
+ encryption
  reference
  news
 

docs/presentations/demo_encryption.py

@@ -0,0 +1,84 @@
+import bson
+from ming import Session
+from ming.datastore import create_engine, create_datastore, DataStore
+from ming.encryption import EncryptionConfig
+import ming.schema as S
+from ming.tests import make_encryption_key
+
+bind: DataStore = create_datastore(
+ 'mongodb://localhost:27017/test_database',
+ encryption=EncryptionConfig({
+ 'kms_providers': {
+ 'local': {
+ # Don't use this for production! This is just for demo purposes
+ 'key': make_encryption_key('demo_encryption'),
+ },
+ },
+ 'key_vault_namespace': 'demo_encryption_db.__keyVault',
+ 'provider_options': {
+ 'local': {
+ 'key_alt_names': ['datakeyName'],
+ },
+ },
+ }))
+
+# clean up for our demo purposes
+bind.conn.drop_database('test_database')
+bind.conn.drop_database('demo_encryption_db')
+
+session = Session(bind)
+
+from ming import Field, Document, schema
+from ming.encryption import DecryptedField
+import datetime
+
+class UserEmail(Document):
+ class __mongometa__:
+ session = session
+ name = 'user_emails'
+ _id = Field(schema.ObjectId)
+
+ # Encrypted fields should:
+ # - Have '_encrypted' suffix
+ # - Have type Binary
+ email_encrypted = Field(S.Binary, if_missing=None)
+
+ # Decrypted fields should:
+ # - Have no suffix
+ # - Have the actual type
+ # - Provide the encrypted field's full name
+ email = DecryptedField(str, 'email_encrypted')
+
+
+user_email = UserEmail.make({})
+assert not user_email.email
+assert not user_email.email_encrypted
+
+# Can directly set DecryptedField and it will auto-populate and encrypt its counterpart
+user_email.email = 'rick@example.com'
+assert user_email.email_encrypted is not None
+assert user_email.email_encrypted != 'rick@example.com'
+assert isinstance(user_email.email_encrypted, bson.Binary)
+user_email.m.save()
+
+
+# Use .make_encr to properly create new instance with unencrypted data
+user_email2 = UserEmail.make_encr(dict(
+ email='stacy@example.com'))
+
+
+assert user_email2.email_encrypted is not None
+assert user_email2.email_encrypted != 'stacy@example.com'
+assert isinstance(user_email2.email_encrypted, bson.Binary)
+blob1 = user_email2.email_encrypted
+
+user_email2.m.save()
+
+# updating the email updates the corresponding encrypted field
+user_email2.email = 'stacy+1@example.com'
+assert user_email2.email_encrypted != blob1
+
+user_email2.m.save()
+
+bind.conn.drop_database('test_database')
+bind.conn.drop_database('demo_encryption_db')

ming/config.py

@@ -24,6 +24,7 @@ def configure(**kwargs):
 def configure_from_nested_dict(config):
  try:
  from formencode import schema, validators
+ import ming.validators as ming_validators
  except ImportError:
  raise MingConfigError("Need to install FormEncode to use ``ming.configure``")
 
@@ -36,6 +37,7 @@ class DatastoreSchema(schema.Schema):
  auto_ensure_indexes = validators.StringBool(if_missing=True)
  # pymongo
  tz_aware = validators.Bool(if_missing=False)
+ encryption = ming_validators.EncryptionConfigValidator(if_missing=None)
 
  datastores = {}
  for name, datastore in config.items():

ming/datastore.py

@@ -1,7 +1,9 @@
+from __future__ import annotations
+
 import time
 import logging
 from threading import Lock
-from typing import Union
+from typing import Union, TYPE_CHECKING
 import urllib
 from pymongo import MongoClient
 from pymongo.database import Database
@@ -11,10 +13,13 @@
 from . import mim
 from . import exc
 
+if TYPE_CHECKING:
+ from . import encryption
+
 Conn = Union[mim.Connection, MongoClient]
 
 
-def create_engine(*args, **kwargs):
+def create_engine(*args, **kwargs) -> Engine:
  """Creates a new :class:`.Engine` instance.
 
  According to the provided url schema ``mongodb://`` or ``mim://``
@@ -34,7 +39,7 @@
  return Engine(use_class, args, kwargs, connect_retry, auto_ensure_indexes)
 
 
-def create_datastore(uri, **kwargs):
+def create_datastore(uri, **kwargs) -> DataStore:
  """Creates a new :class:`.DataStore` for the database identified by ``uri``.
 
  ``uri`` is a mongodb url in the form ``mongodb://username:password@address:port/dbname``,
@@ -74,6 +79,8 @@
  if database.startswith("/"):
  database = database[1:]
 
+ encryption_config: encryption.EncryptionConfig = kwargs.pop('encryption', None)
+
  if uri:
  # User provided a valid connection URL.
  if bind:
@@ -85,14 +92,14 @@
  # Create engine without connection.
  bind = create_engine(**kwargs)
 
- return DataStore(bind, database)
+ return DataStore(bind, database, encryption_config)
 
 
 class Engine:
  """Engine represents the connection to a MongoDB (or in-memory database).
 
- The ``Engine`` class lazily creates the connection the firs time it's
- actually accessed.
+ The ``Engine`` class lazily creates the connection the first time it's
+ accessed.
  """
 
  def __init__(self, Connection,
@@ -135,6 +142,7 @@
  try:
  with self._lock:
  if self._conn is None:
+ # NOTE: Runs MongoClient/EncryptionClient
  self._conn = self._Connection(
  *self._conn_args, **self._conn_kwargs)
  else:
@@ -159,10 +167,10 @@
  :func:`.create_datastore` function.
  """
 
- def __init__(self, bind, name, authenticate=None):
+ def __init__(self, bind: Engine, name: str, encryption_config: encryption.EncryptionConfig = None):
  self.bind = bind
  self.name = name
- self._authenticate = authenticate
+ self._encryption_config = encryption_config
  self._db = None
 
  def __repr__(self): # pragma no cover
@@ -191,3 +199,7 @@
 
  self._db = self.bind[self.name]
  return self._db
+
+ @property
+ def encryption(self) -> encryption.EncryptionConfig | None:
+ return self._encryption_config

ming/declarative.pyi

@@ -1,6 +1,9 @@
 from typing import TypeVar, Mapping, Any
 
+from pymongo.encryption import ClientEncryption
+
 from ming.base import Object
+from ming.datastore import DataStore
 from ming.metadata import Manager
 
 M = TypeVar('M')
@@ -16,6 +19,34 @@ class Document(Object):
  @classmethod
  def make(cls, data, allow_extra=False, strip_extra=True) -> Document: ...
 
+ # Encryption-Related fields:
+
+ @classmethod
+ def make_encr(cls, data: dict) -> Document: ...
+
+ @classmethod
+ def encr(cls, s: str | None, _first_attempt=True, provider='local') -> bytes | None: ...
+
+ @classmethod
+ def decr(cls, b: bytes | None) -> str | None: ...
+
+ @classmethod
+ def encryptor(cls, ming_ds: DataStore) -> ClientEncryption: ...
+
+ @classmethod
+ def make_data_key(cls): ...
+
+ @classmethod
+ def decrypted_field_names(cls) -> list[str]: ...
+
+ @classmethod
+ def encrypted_field_names(cls) -> list[str]: ...
+
+ @classmethod
+ def encrypt_some_fields(cls, data: dict) -> dict: ...
+
+ def decrypt_some_fields(self, data: dict) -> dict: ...
+
  # ...
  # class __mongometa__:
  # name: Any = ...