DRIVERS-2732 CSFLE/QE KMIP support for encrypt/decrypt

source/client-side-encryption/client-side-encryption.md

@@ -1087,7 +1087,8 @@ If the kmsProvider has KMS provider type "kmip", the masterKey is required and h
 {
    keyId: Optional<String>, // keyId is the KMIP Unique Identifier to a 96 byte KMIP Secret Data managed object.
                             // If keyId is omitted, the driver creates a random 96 byte KMIP Secret Data managed object.
-   endpoint: Optional<String> // Host with optional port.
+   endpoint: Optional<String>, // Host with optional port.
+   delegated: Optional<Boolean> // If true, this key should be decrypted by the KMIP server.
 }
 ```
 
@@ -1727,12 +1728,13 @@ Data keys are stored in the MongoDB key vault collection with the following sche
 | **Name** | **Type** | **Description** |
 | provider | "local"  |                 |
 
-|          |          |                                                                                        |
-| -------- | -------- | -------------------------------------------------------------------------------------- |
-| **Name** | **Type** | **Description**                                                                        |
-| provider | "kmip"   |                                                                                        |
-| endpoint | String   | Optional. Defaults to kmip.endpoint from KMS providers.                                |
-| keyId    | String   | Required. keyId is the Unique Identifier to a 96 byte KMIP Secret Data managed object. |
+|           |          |                                                                                        |
+| --------- | -------- | -------------------------------------------------------------------------------------- |
+| **Name**  | **Type** | **Description**                                                                        |
+| provider  | "kmip"   |                                                                                        |
+| endpoint  | String   | Optional. Defaults to kmip.endpoint from KMS providers.                                |
+| delegated | Boolean  | Optional. Defaults to false.                                                           |
+| keyId     | String   | Required. keyId is the Unique Identifier to a 96 byte KMIP Secret Data managed object. |
 
 Data keys are needed for encryption and decryption. They are identified in the intent-to-encrypt marking and ciphertext.
 Data keys may be retrieved by querying the "\_id" with a UUID or by querying the "keyAltName" with a string.
@@ -2327,6 +2329,12 @@ KMS credentials would require added work in drivers inspecting the KMS providers
 additional test coverage. Supporting on-demand KMS credentials for named KMS providers can be considered as future work
 if needed.
 
+### What is the KMIP `delegated` option?
+
+By default, the KMS will retrieve the key encryption key from the KMIP server and use it to encrypt the data key. If the
+`delegated` option is set to true (recommended), the KMIP server will instead perform encryption and decryption locally,
+ensuring that the key encryption key never leaves the server.
+
 ## Future work
 
 ### Make libmonogocrypt cache window configurable
@@ -2381,6 +2389,8 @@ explicit session parameter as described in the [Drivers Sessions Specification](
 
 ## Changelog
 
+- 2024-03-20: Add `delegated` option to "kmip" KMS provider
+
 - 2024-02-27: Migrated from reStructuredText to Markdown.
 
 - 2024-01-10: Add named KMS providers

source/client-side-encryption/etc/generate-test.py

@@ -179,6 +179,35 @@
             "keyId": "1"
         },
         "keyAltNames": ["altname", "kmip_altname"]
+    },
+    "kmip_delegated": {
+        "_id": {
+            "$uuid": "7411e9af-c688-4df7-8143-5e60ae96cba6"
+        },
+        "keyMaterial": {
+            "$binary": {
+                "base64": "5TLMFWlguBWe5GUESTvOVtkdBsCrynhnV72XRyZ66/nk+EP9/1oEp1t1sg0+vwCTqULHjBiUE6DRx2mYD/Eup1+u2Jgz9/+1sV1drXeOPALNPkSgiZiDbIb67zRi+wTABEcKcegJH+FhmSGxwUoQAiHCsCbcvia5P8tN1lt98YQ=", "subType": "00"
+            }
+        },
+        "creationDate": {
+            "$date": {
+                "$numberLong": "1634220190041"
+            }
+        },
+        "updateDate": {
+            "$date": {
+                "$numberLong": "1634220190041"
+            }
+        },
+        "status": {
+            "$numberInt": "0"
+        },
+        "masterKey": {
+            "provider": "kmip",
+            "delegated": True,
+            "keyId": "11"
+        },
+        "keyAltNames": ["delegated"]
     }
 }
 
@@ -254,6 +283,11 @@
                     "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
                 }
             },
+            "encrypted_string_kmip_delegated": {
+                "encrypt": {
+                    "keyId": [keys["kmip_delegated"]["_id"]], 
+                    "bsonType": "string",
+                    "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"}},
         },
         "bsonType": "object"
     },
@@ -563,6 +597,17 @@
             }
         }
     },
+    {
+        "schema": "all",
+        "field": "encrypted_string_kmip_delegated",
+        "plaintext": "string0",
+        "data": {
+            "$binary": {
+                "base64": "AXQR6a/GiE33gUNeYK6Wy6YCkB+8NVfAAjIbvLqyXIg6g1a8tXrym92DPoqmxpcdQyH0vQM3aFNMz7tZwQBimKs29ztZV/LWjM633HhO5ACl9A==",
+                "subType": "06"
+            }
+        }
+    },
     {
         "schema": "local:name2",
         "field": "encrypted_string",

source/client-side-encryption/etc/test-templates/kmipKMS.yml.template

@@ -5,7 +5,7 @@ collection_name: &collection_name "default"
 
 data: []
 json_schema: {{schema("all")}}
-key_vault_data: [{{key("kmip")}}]
+key_vault_data: [{{key("kmip")}},{{key("kmip_delegated")}}]
 
 tests:
   - description: "Insert a document with auto encryption using KMIP KMS provider"
@@ -43,4 +43,47 @@ tests:
       collection:
         # Outcome is checked using a separate MongoClient without auto encryption.
         data:
-          - *doc0_encrypted
+          - *doc0_encrypted
+
+  - description: "Insert a document with auto encryption using KMIP delegated KMS provider"
+    clientOptions:
+      autoEncryptOpts:
+        kmsProviders:
+          kmip: {}
+    operations:
+      - name: insertOne
+        arguments:
+          document: &doc1 { _id: 1, encrypted_string_kmip_delegated: "string0" }
+    expectations:
+    - command_started_event:
+        command:
+          listCollections: 1
+          filter:
+            name: *collection_name
+        command_name: listCollections
+    # Auto encryption will request the collection info.
+    - command_started_event:
+        command:
+          listCollections: 1
+          filter:
+            name: datakeys
+        command_name: listCollections
+    # Then key is fetched from the key vault.
+    - command_started_event:
+        command:
+          find: datakeys
+          filter: { $or: [ { _id: { $in: [ {{key("kmip_delegated")["_id"]}} ] } }, { keyAltNames: { $in: [] } } ] }
+          $db: keyvault
+        command_name: find
+    - command_started_event:
+        command:
+          insert: *collection_name
+          documents:
+            - &doc1_encrypted { _id: 1, encrypted_string_kmip_delegated: {{ciphertext("string0", field="encrypted_string_kmip_delegated", schema="all")}} }
+          ordered: true
+        command_name: insert
+    outcome:
+      collection:
+        # Outcome is checked using a separate MongoClient without auto encryption.
+        data:
+          - *doc1_encrypted

source/client-side-encryption/tests/legacy/azureKMS.yml

@@ -4,7 +4,7 @@ database_name: &database_name "default"
 collection_name: &collection_name "default"
 
 data: []
-json_schema: {'properties': {'encrypted_string_aws': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'encrypted_string_azure': {'encrypt': {'keyId': [{'$binary': {'base64': 'AZURE+AAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'encrypted_string_gcp': {'encrypt': {'keyId': [{'$binary': {'base64': 'GCP+AAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'encrypted_string_local': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'encrypted_string_kmip': {'encrypt': {'keyId': [{'$binary': {'base64': 'dBHpr8aITfeBQ15grpbLpQ==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}}, 'bsonType': 'object'}
+json_schema: {'properties': {'encrypted_string_aws': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'encrypted_string_azure': {'encrypt': {'keyId': [{'$binary': {'base64': 'AZURE+AAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'encrypted_string_gcp': {'encrypt': {'keyId': [{'$binary': {'base64': 'GCP+AAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'encrypted_string_local': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'encrypted_string_kmip': {'encrypt': {'keyId': [{'$binary': {'base64': 'dBHpr8aITfeBQ15grpbLpQ==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'encrypted_string_kmip_delegated': {'encrypt': {'keyId': [{'$uuid': '7411e9af-c688-4df7-8143-5e60ae96cba6'}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}}, 'bsonType': 'object'}
 key_vault_data: [{'_id': {'$binary': {'base64': 'AZURE+AAAAAAAAAAAAAAAA==', 'subType': '04'}}, 'keyMaterial': {'$binary': {'base64': 'n+HWZ0ZSVOYA3cvQgP7inN4JSXfOH85IngmeQxRpQHjCCcqT3IFqEWNlrsVHiz3AELimHhX4HKqOLWMUeSIT6emUDDoQX9BAv8DR1+E1w4nGs/NyEneac78EYFkK3JysrFDOgl2ypCCTKAypkn9CkAx1if4cfgQE93LW4kczcyHdGiH36CIxrCDGv1UzAvERN5Qa47DVwsM6a+hWsF2AAAJVnF0wYLLJU07TuRHdMrrphPWXZsFgyV+lRqJ7DDpReKNO8nMPLV/mHqHBHGPGQiRdb9NoJo8CvokGz4+KE8oLwzKf6V24dtwZmRkrsDV4iOhvROAzz+Euo1ypSkL3mw==', 'subType': '00'}}, 'creationDate': {'$date': {'$numberLong': '1601573901680'}}, 'updateDate': {'$date': {'$numberLong': '1601573901680'}}, 'status': {'$numberInt': '0'}, 'masterKey': {'provider': 'azure', 'keyVaultEndpoint': 'key-vault-csfle.vault.azure.net', 'keyName': 'key-name-csfle'}, 'keyAltNames': ['altname', 'azure_altname']}]
 
 tests:

source/client-side-encryption/tests/legacy/gcpKMS.yml

@@ -4,7 +4,7 @@ database_name: &database_name "default"
 collection_name: &collection_name "default"
 
 data: []
-json_schema: {'properties': {'encrypted_string_aws': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'encrypted_string_azure': {'encrypt': {'keyId': [{'$binary': {'base64': 'AZURE+AAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'encrypted_string_gcp': {'encrypt': {'keyId': [{'$binary': {'base64': 'GCP+AAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'encrypted_string_local': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'encrypted_string_kmip': {'encrypt': {'keyId': [{'$binary': {'base64': 'dBHpr8aITfeBQ15grpbLpQ==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}}, 'bsonType': 'object'}
+json_schema: {'properties': {'encrypted_string_aws': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'encrypted_string_azure': {'encrypt': {'keyId': [{'$binary': {'base64': 'AZURE+AAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'encrypted_string_gcp': {'encrypt': {'keyId': [{'$binary': {'base64': 'GCP+AAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'encrypted_string_local': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'encrypted_string_kmip': {'encrypt': {'keyId': [{'$binary': {'base64': 'dBHpr8aITfeBQ15grpbLpQ==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'encrypted_string_kmip_delegated': {'encrypt': {'keyId': [{'$uuid': '7411e9af-c688-4df7-8143-5e60ae96cba6'}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}}, 'bsonType': 'object'}
 key_vault_data: [{'_id': {'$binary': {'base64': 'GCP+AAAAAAAAAAAAAAAAAA==', 'subType': '04'}}, 'keyMaterial': {'$binary': {'base64': 'CiQAIgLj0WyktnB4dfYHo5SLZ41K4ASQrjJUaSzl5vvVH0G12G0SiQEAjlV8XPlbnHDEDFbdTO4QIe8ER2/172U1ouLazG0ysDtFFIlSvWX5ZnZUrRMmp/R2aJkzLXEt/zf8Mn4Lfm+itnjgo5R9K4pmPNvvPKNZX5C16lrPT+aA+rd+zXFSmlMg3i5jnxvTdLHhg3G7Q/Uv1ZIJskKt95bzLoe0tUVzRWMYXLIEcohnQg==', 'subType': '00'}}, 'creationDate': {'$date': {'$numberLong': '1601574333107'}}, 'updateDate': {'$date': {'$numberLong': '1601574333107'}}, 'status': {'$numberInt': '0'}, 'masterKey': {'provider': 'gcp', 'projectId': 'devprod-drivers', 'location': 'global', 'keyRing': 'key-ring-csfle', 'keyName': 'key-name-csfle'}, 'keyAltNames': ['altname', 'gcp_altname']}]
 
 tests: