test(NODE-6098): enable unified valid fail tests

test/integration/unified-test-format/unified_test_format.spec.test.ts

@@ -41,7 +41,11 @@ const filter: TestFilter = ({ description }) => {
   return false;
 };
 
-describe('Unified test format runner', function unifiedTestRunner() {
+describe('Unified test format runner (valid-pass)', function unifiedTestRunner() {
   // Valid tests that should pass
   runUnifiedSuite(loadSpecTests('unified-test-format/valid-pass'), filter);
 });
+
+describe('Unified test format runner (valid-fail)', function unifiedTestRunner() {
+  runUnifiedSuite(loadSpecTests('unified-test-format/valid-fail'), () => false, true);
+});

test/spec/unified-test-format/valid-fail/entity-findCursor-malformed.json

@@ -0,0 +1,44 @@
+{
+  "description": "entity-findCursor-malformed",
+  "schemaVersion": "1.3",
+  "createEntities": [
+    {
+      "client": {
+        "id": "client0"
+      }
+    },
+    {
+      "database": {
+        "id": "database0",
+        "client": "client0",
+        "databaseName": "database0Name"
+      }
+    },
+    {
+      "collection": {
+        "id": "collection0",
+        "database": "database0",
+        "collectionName": "coll0"
+      }
+    }
+  ],
+  "initialData": [
+    {
+      "databaseName": "database0Name",
+      "collectionName": "coll0",
+      "documents": []
+    }
+  ],
+  "tests": [
+    {
+      "description": "createFindCursor fails if filter is not specified",
+      "operations": [
+        {
+          "name": "createFindCursor",
+          "object": "collection0",
+          "saveResultAsEntity": "cursor0"
+        }
+      ]
+    }
+  ]
+}

test/spec/unified-test-format/valid-fail/entity-findCursor-malformed.yml

@@ -0,0 +1,31 @@
+# This test is split out into a separate file to accommodate drivers that validate operation structure while decoding
+# from JSON/YML. Such drivers fail to decode any files containing invalid operations. Combining this test in a file
+# with other entity-findCursor valid-fail tests, which test failures that occur during test execution, would prevent
+# such drivers from decoding the file and running any of the tests.
+description: entity-findCursor-malformed
+
+schemaVersion: '1.3'
+
+createEntities:
+  - client:
+      id: &client0 client0
+  - database:
+      id: &database0 database0
+      client: *client0
+      databaseName: &database0Name database0Name
+  - collection:
+      id: &collection0 collection0
+      database: *database0
+      collectionName: &collection0Name coll0
+
+initialData:
+  - databaseName: *database0Name
+    collectionName: *collection0Name
+    documents: []
+
+tests:
+  - description: createFindCursor fails if filter is not specified
+    operations:
+      - name: createFindCursor
+        object: *collection0
+        saveResultAsEntity: &cursor0 cursor0

test/spec/unified-test-format/valid-fail/entity-findCursor.json

@@ -0,0 +1,52 @@
+{
+  "description": "entity-findCursor",
+  "schemaVersion": "1.3",
+  "createEntities": [
+    {
+      "client": {
+        "id": "client0"
+      }
+    },
+    {
+      "database": {
+        "id": "database0",
+        "client": "client0",
+        "databaseName": "database0Name"
+      }
+    },
+    {
+      "collection": {
+        "id": "collection0",
+        "database": "database0",
+        "collectionName": "coll0"
+      }
+    }
+  ],
+  "initialData": [
+    {
+      "databaseName": "database0Name",
+      "collectionName": "coll0",
+      "documents": []
+    }
+  ],
+  "tests": [
+    {
+      "description": "iterateUntilDocumentOrError fails if it references a nonexistent entity",
+      "operations": [
+        {
+          "name": "iterateUntilDocumentOrError",
+          "object": "cursor0"
+        }
+      ]
+    },
+    {
+      "description": "close fails if it references a nonexistent entity",
+      "operations": [
+        {
+          "name": "close",
+          "object": "cursor0"
+        }
+      ]
+    }
+  ]
+}

test/spec/unified-test-format/valid-fail/entity-findCursor.yml

@@ -0,0 +1,31 @@
+description: entity-findCursor
+
+schemaVersion: '1.3'
+
+createEntities:
+  - client:
+      id: &client0 client0
+  - database:
+      id: &database0 database0
+      client: *client0
+      databaseName: &database0Name database0Name
+  - collection:
+      id: &collection0 collection0
+      database: *database0
+      collectionName: &collection0Name coll0
+
+initialData:
+  - databaseName: *database0Name
+    collectionName: *collection0Name
+    documents: []
+
+tests:
+  - description: iterateUntilDocumentOrError fails if it references a nonexistent entity
+    operations:
+      - name: iterateUntilDocumentOrError
+        object: cursor0
+
+  - description: close fails if it references a nonexistent entity
+    operations:
+      - name: close
+        object: cursor0

test/spec/unified-test-format/valid-fail/ignoreResultAndError-malformed.json

@@ -0,0 +1,48 @@
+{
+  "description": "ignoreResultAndError-malformed",
+  "schemaVersion": "1.3",
+  "createEntities": [
+    {
+      "client": {
+        "id": "client0",
+        "useMultipleMongoses": true
+      }
+    },
+    {
+      "database": {
+        "id": "database0",
+        "client": "client0",
+        "databaseName": "database0Name"
+      }
+    },
+    {
+      "collection": {
+        "id": "collection0",
+        "database": "database0",
+        "collectionName": "coll0"
+      }
+    }
+  ],
+  "initialData": [
+    {
+      "collectionName": "coll0",
+      "databaseName": "database0Name",
+      "documents": []
+    }
+  ],
+  "tests": [
+    {
+      "description": "malformed operation fails if ignoreResultAndError is true",
+      "operations": [
+        {
+          "name": "insertOne",
+          "object": "collection0",
+          "arguments": {
+            "foo": "bar"
+          },
+          "ignoreResultAndError": true
+        }
+      ]
+    }
+  ]
+}

test/spec/unified-test-format/valid-fail/ignoreResultAndError-malformed.yml

@@ -0,0 +1,34 @@
+# This test is split out into a separate file to accommodate drivers that validate operation structure while decoding
+# from JSON/YML. Such drivers fail to decode any files containing invalid operations. Combining this test in a file
+# with other ignoreResultAndError valid-fail tests, which test failures that occur during test execution, would prevent
+# such drivers from decoding the file and running any of the tests.
+description: ignoreResultAndError-malformed  
+
+schemaVersion: '1.3'
+
+createEntities:
+  - client:
+      id: &client0 client0
+      useMultipleMongoses: true
+  - database:
+      id: &database0 database0
+      client: *client0
+      databaseName: &database0Name database0Name
+  - collection:
+      id: &collection0 collection0
+      database: *database0
+      collectionName: &collection0Name coll0
+
+initialData:
+  - collectionName: *collection0Name
+    databaseName: *database0Name
+    documents: []
+
+tests:
+  - description: malformed operation fails if ignoreResultAndError is true
+    operations:
+      - name: insertOne
+        object: *collection0
+        arguments:
+          foo: bar
+        ignoreResultAndError: true

test/spec/unified-test-format/valid-fail/ignoreResultAndError.json

@@ -54,19 +54,6 @@
           "ignoreResultAndError": false
         }
       ]
-    },
-    {
-      "description": "malformed operation fails if ignoreResultAndError is true",
-      "operations": [
-        {
-          "name": "insertOne",
-          "object": "collection0",
-          "arguments": {
-            "foo": "bar"
-          },
-          "ignoreResultAndError": true
-        }
-      ]
     }
   ]
 }

test/spec/unified-test-format/valid-fail/ignoreResultAndError.yml

@@ -33,11 +33,3 @@ tests:
           # Insert the same document to force a DuplicateKey error.
           document: *insertDocument
         ignoreResultAndError: false
-
-  - description: malformed operation fails if ignoreResultAndError is true
-    operations:
-      - name: insertOne
-        object: *collection0
-        arguments:
-          foo: bar
-        ignoreResultAndError: true

test/spec/unified-test-format/valid-fail/operation-unsupported.json

@@ -0,0 +1,22 @@
+{
+  "description": "operation-unsupported",
+  "schemaVersion": "1.0",
+  "createEntities": [
+    {
+      "client": {
+        "id": "client0"
+      }
+    }
+  ],
+  "tests": [
+    {
+      "description": "Unsupported operation",
+      "operations": [
+        {
+          "name": "unsupportedOperation",
+          "object": "client0"
+        }
+      ]
+    }
+  ]
+}

test/spec/unified-test-format/valid-fail/operation-unsupported.yml

@@ -0,0 +1,13 @@
+description: "operation-unsupported"
+
+schemaVersion: "1.0"
+
+createEntities:
+  - client:
+      id: &client0 client0
+
+tests:
+  - description: "Unsupported operation"
+    operations:
+      - name: unsupportedOperation
+        object: *client0

test/tools/unified-spec-runner/entities.ts

@@ -586,11 +586,17 @@ export class EntitiesMap<E = Entity> extends Map<string, E> {
           uri = makeConnectionString(config.url({ useMultipleMongoses }), entity.client.uriOptions);
         }
         const client = new UnifiedMongoClient(uri, entity.client);
-        new EntityEventRegistry(client, entity.client, map).register();
         try {
+          new EntityEventRegistry(client, entity.client, map).register();
           await client.connect();
         } catch (error) {
           console.error(ejson`failed to connect entity ${entity}`);
+          // In the case where multiple clients are defined in the test and any one of them failed
+          // to connect, but others did succeed, we need to ensure all open clients are closed.
+          const clients = map.mapOf('client');
+          for (const clientEntry of clients.values()) {
+            await clientEntry.close();
+          }
           throw error;
         }
         map.set(entity.client.id, client);
@@ -668,7 +674,7 @@ export class EntitiesMap<E = Entity> extends Map<string, E> {
       } else if ('stream' in entity) {
         throw new Error(`Unsupported Entity ${JSON.stringify(entity)}`);
       } else if ('clientEncryption' in entity) {
-        const clientEncryption = createClientEncryption(map, entity.clientEncryption);
+        const clientEncryption = await createClientEncryption(map, entity.clientEncryption);
 
         map.set(entity.clientEncryption.id, clientEncryption);
       } else {

test/tools/unified-spec-runner/entity_event_registry.ts

@@ -60,14 +60,20 @@ export class EntityEventRegistry {
   register(): void {
     if (this.clientEntity.storeEventsAsEntities) {
       for (const { id, events } of this.clientEntity.storeEventsAsEntities) {
+        if (this.entitiesMap.has(id)) {
+          throw new Error(`Duplicate id ${id} found while storing events as entities`);
+        }
         this.entitiesMap.set(id, []);
         for (const eventName of events) {
           // Need to map the event names to the Node event names.
           this.client.on(MAPPINGS[eventName], () => {
-            this.entitiesMap.getEntity('events', id).push({
-              name: eventName,
-              observedAt: Date.now()
-            });
+            const events = this.entitiesMap.getEntity('events', id);
+            if (events.push) {
+              events.push({
+                name: eventName,
+                observedAt: Date.now()
+              });
+            }
           });
         }
       }