test(NODE-6098): enable unified valid fail tests

test/integration/unified-test-format/unified_test_format.spec.test.ts

@@ -41,7 +41,11 @@ const filter: TestFilter = ({ description }) => {
  return false;
 };
 
-describe('Unified test format runner', function unifiedTestRunner() {
+describe('Unified test format runner (valid-pass)', function unifiedTestRunner() {
  // Valid tests that should pass
  runUnifiedSuite(loadSpecTests('unified-test-format/valid-pass'), filter);
 });
+
+describe('Unified test format runner (valid-fail)', function unifiedTestRunner() {
+ runUnifiedSuite(loadSpecTests('unified-test-format/valid-fail'), () => false, true);
+});

test/spec/unified-test-format/valid-fail/entity-findCursor-malformed.json

@@ -0,0 +1,44 @@
+{
+ "description": "entity-findCursor-malformed",
+ "schemaVersion": "1.3",
+ "createEntities": [
+ {
+ "client": {
+ "id": "client0"
+ }
+ },
+ {
+ "database": {
+ "id": "database0",
+ "client": "client0",
+ "databaseName": "database0Name"
+ }
+ },
+ {
+ "collection": {
+ "id": "collection0",
+ "database": "database0",
+ "collectionName": "coll0"
+ }
+ }
+ ],
+ "initialData": [
+ {
+ "databaseName": "database0Name",
+ "collectionName": "coll0",
+ "documents": []
+ }
+ ],
+ "tests": [
+ {
+ "description": "createFindCursor fails if filter is not specified",
+ "operations": [
+ {
+ "name": "createFindCursor",
+ "object": "collection0",
+ "saveResultAsEntity": "cursor0"
+ }
+ ]
+ }
+ ]
+}

test/spec/unified-test-format/valid-fail/entity-findCursor-malformed.yml

@@ -0,0 +1,31 @@
+# This test is split out into a separate file to accommodate drivers that validate operation structure while decoding
+# from JSON/YML. Such drivers fail to decode any files containing invalid operations. Combining this test in a file
+# with other entity-findCursor valid-fail tests, which test failures that occur during test execution, would prevent
+# such drivers from decoding the file and running any of the tests.
+description: entity-findCursor-malformed
+
+schemaVersion: '1.3'
+
+createEntities:
+ - client:
+ id: &client0 client0
+ - database:
+ id: &database0 database0
+ client: *client0
+ databaseName: &database0Name database0Name
+ - collection:
+ id: &collection0 collection0
+ database: *database0
+ collectionName: &collection0Name coll0
+
+initialData:
+ - databaseName: *database0Name
+ collectionName: *collection0Name
+ documents: []
+
+tests:
+ - description: createFindCursor fails if filter is not specified
+ operations:
+ - name: createFindCursor
+ object: *collection0
+ saveResultAsEntity: &cursor0 cursor0

test/spec/unified-test-format/valid-fail/entity-findCursor.json

@@ -0,0 +1,52 @@
+{
+ "description": "entity-findCursor",
+ "schemaVersion": "1.3",
+ "createEntities": [
+ {
+ "client": {
+ "id": "client0"
+ }
+ },
+ {
+ "database": {
+ "id": "database0",
+ "client": "client0",
+ "databaseName": "database0Name"
+ }
+ },
+ {
+ "collection": {
+ "id": "collection0",
+ "database": "database0",
+ "collectionName": "coll0"
+ }
+ }
+ ],
+ "initialData": [
+ {
+ "databaseName": "database0Name",
+ "collectionName": "coll0",
+ "documents": []
+ }
+ ],
+ "tests": [
+ {
+ "description": "iterateUntilDocumentOrError fails if it references a nonexistent entity",
+ "operations": [
+ {
+ "name": "iterateUntilDocumentOrError",
+ "object": "cursor0"
+ }
+ ]
+ },
+ {
+ "description": "close fails if it references a nonexistent entity",
+ "operations": [
+ {
+ "name": "close",
+ "object": "cursor0"
+ }
+ ]
+ }
+ ]
+}

test/spec/unified-test-format/valid-fail/entity-findCursor.yml

@@ -0,0 +1,31 @@
+description: entity-findCursor
+
+schemaVersion: '1.3'
+
+createEntities:
+ - client:
+ id: &client0 client0
+ - database:
+ id: &database0 database0
+ client: *client0
+ databaseName: &database0Name database0Name
+ - collection:
+ id: &collection0 collection0
+ database: *database0
+ collectionName: &collection0Name coll0
+
+initialData:
+ - databaseName: *database0Name
+ collectionName: *collection0Name
+ documents: []
+
+tests:
+ - description: iterateUntilDocumentOrError fails if it references a nonexistent entity
+ operations:
+ - name: iterateUntilDocumentOrError
+ object: cursor0
+
+ - description: close fails if it references a nonexistent entity
+ operations:
+ - name: close
+ object: cursor0

test/spec/unified-test-format/valid-fail/ignoreResultAndError-malformed.json

@@ -0,0 +1,48 @@
+{
+ "description": "ignoreResultAndError-malformed",
+ "schemaVersion": "1.3",
+ "createEntities": [
+ {
+ "client": {
+ "id": "client0",
+ "useMultipleMongoses": true
+ }
+ },
+ {
+ "database": {
+ "id": "database0",
+ "client": "client0",
+ "databaseName": "database0Name"
+ }
+ },
+ {
+ "collection": {
+ "id": "collection0",
+ "database": "database0",
+ "collectionName": "coll0"
+ }
+ }
+ ],
+ "initialData": [
+ {
+ "collectionName": "coll0",
+ "databaseName": "database0Name",
+ "documents": []
+ }
+ ],
+ "tests": [
+ {
+ "description": "malformed operation fails if ignoreResultAndError is true",
+ "operations": [
+ {
+ "name": "insertOne",
+ "object": "collection0",
+ "arguments": {
+ "foo": "bar"
+ },
+ "ignoreResultAndError": true
+ }
+ ]
+ }
+ ]
+}

test/spec/unified-test-format/valid-fail/ignoreResultAndError-malformed.yml

@@ -0,0 +1,34 @@
+# This test is split out into a separate file to accommodate drivers that validate operation structure while decoding
+# from JSON/YML. Such drivers fail to decode any files containing invalid operations. Combining this test in a file
+# with other ignoreResultAndError valid-fail tests, which test failures that occur during test execution, would prevent
+# such drivers from decoding the file and running any of the tests.
+description: ignoreResultAndError-malformed 
+
+schemaVersion: '1.3'
+
+createEntities:
+ - client:
+ id: &client0 client0
+ useMultipleMongoses: true
+ - database:
+ id: &database0 database0
+ client: *client0
+ databaseName: &database0Name database0Name
+ - collection:
+ id: &collection0 collection0
+ database: *database0
+ collectionName: &collection0Name coll0
+
+initialData:
+ - collectionName: *collection0Name
+ databaseName: *database0Name
+ documents: []
+
+tests:
+ - description: malformed operation fails if ignoreResultAndError is true
+ operations:
+ - name: insertOne
+ object: *collection0
+ arguments:
+ foo: bar
+ ignoreResultAndError: true

test/spec/unified-test-format/valid-fail/ignoreResultAndError.json

@@ -54,19 +54,6 @@
  "ignoreResultAndError": false
  }
  ]
- },
- {
- "description": "malformed operation fails if ignoreResultAndError is true",
- "operations": [
- {
- "name": "insertOne",
- "object": "collection0",
- "arguments": {
- "foo": "bar"
- },
- "ignoreResultAndError": true
- }
- ]
  }
  ]
 }

test/spec/unified-test-format/valid-fail/ignoreResultAndError.yml

@@ -33,11 +33,3 @@ tests:
  # Insert the same document to force a DuplicateKey error.
  document: *insertDocument
  ignoreResultAndError: false
-
- - description: malformed operation fails if ignoreResultAndError is true
- operations:
- - name: insertOne
- object: *collection0
- arguments:
- foo: bar
- ignoreResultAndError: true

test/spec/unified-test-format/valid-fail/operation-unsupported.json

@@ -0,0 +1,22 @@
+{
+ "description": "operation-unsupported",
+ "schemaVersion": "1.0",
+ "createEntities": [
+ {
+ "client": {
+ "id": "client0"
+ }
+ }
+ ],
+ "tests": [
+ {
+ "description": "Unsupported operation",
+ "operations": [
+ {
+ "name": "unsupportedOperation",
+ "object": "client0"
+ }
+ ]
+ }
+ ]
+}

test/spec/unified-test-format/valid-fail/operation-unsupported.yml

@@ -0,0 +1,13 @@
+description: "operation-unsupported"
+
+schemaVersion: "1.0"
+
+createEntities:
+ - client:
+ id: &client0 client0
+
+tests:
+ - description: "Unsupported operation"
+ operations:
+ - name: unsupportedOperation
+ object: *client0

test/tools/unified-spec-runner/entities.ts

@@ -586,11 +586,17 @@ export class EntitiesMap<E = Entity> extends Map<string, E> {
  uri = makeConnectionString(config.url({ useMultipleMongoses }), entity.client.uriOptions);
  }
  const client = new UnifiedMongoClient(uri, entity.client);
- new EntityEventRegistry(client, entity.client, map).register();
  try {
+ new EntityEventRegistry(client, entity.client, map).register();
  await client.connect();
  } catch (error) {
  console.error(ejson`failed to connect entity ${entity}`);
+ // In the case where multiple clients are defined in the test and any one of them failed
+ // to connect, but others did succeed, we need to ensure all open clients are closed.
+ const clients = map.mapOf('client');
+ for (const clientEntry of clients.values()) {
+ await clientEntry.close();
+ }
  throw error;
  }
  map.set(entity.client.id, client);
@@ -668,7 +674,7 @@ export class EntitiesMap<E = Entity> extends Map<string, E> {
  } else if ('stream' in entity) {
  throw new Error(`Unsupported Entity ${JSON.stringify(entity)}`);
  } else if ('clientEncryption' in entity) {
- const clientEncryption = createClientEncryption(map, entity.clientEncryption);
+ const clientEncryption = await createClientEncryption(map, entity.clientEncryption);
 
  map.set(entity.clientEncryption.id, clientEncryption);
  } else {

test/tools/unified-spec-runner/entity_event_registry.ts

@@ -60,6 +60,9 @@ export class EntityEventRegistry {
  register(): void {
  if (this.clientEntity.storeEventsAsEntities) {
  for (const { id, events } of this.clientEntity.storeEventsAsEntities) {
+ if (this.entitiesMap.has(id)) {
+ throw new Error(`Duplicate id ${id} found while storing events as entities`);
+ }
  this.entitiesMap.set(id, []);
  for (const eventName of events) {
  // Need to map the event names to the Node event names.

test/tools/unified-spec-runner/operations.ts

@@ -44,6 +44,8 @@ type RunOperationFn = (
 ) => Promise<Document | boolean | number | null | void | string>;
 export const operations = new Map<string, RunOperationFn>();
 
+export class MalformedOperationError extends Error {}
+
 operations.set('createEntities', async ({ entities, operation, testConfig }) => {
  if (!operation.arguments?.entities) {
  throw new Error('encountered createEntities operation without entities argument');
@@ -356,6 +358,9 @@ operations.set('failPoint', async ({ entities, operation }) => {
 operations.set('insertOne', async ({ entities, operation }) => {
  const collection = entities.getEntity('collection', operation.object);
  const { document, ...opts } = operation.arguments!;
+ if (!document) {
+ throw new MalformedOperationError('No document defined in the arguments for insertOne');
+ }
  // Looping exposes the fact that we can generate _ids for inserted
  // documents and we don't want the original operation to get modified
  // and use the same _id for each insert.
@@ -949,7 +954,7 @@ export async function executeOperationAndCheck(
  if (operation.expectError) {
  expectErrorCheck(error, operation.expectError, entities);
  return;
- } else if (!operation.ignoreResultAndError) {
+ } else if (!operation.ignoreResultAndError || error instanceof MalformedOperationError) {
  throw error;
  }
  }