test(NODE-6098): enable unified valid fail tests (#4181)

test/integration/unified-test-format/unified_test_format.spec.test.ts

@@ -41,7 +41,11 @@ const filter: TestFilter = ({ description }) => {
   return false;
 };
 
-describe('Unified test format runner', function unifiedTestRunner() {
+describe('Unified test format runner (valid-pass)', function unifiedTestRunner() {
   // Valid tests that should pass
   runUnifiedSuite(loadSpecTests('unified-test-format/valid-pass'), filter);
 });
+
+describe('Unified test format runner (valid-fail)', function unifiedTestRunner() {
+  runUnifiedSuite(loadSpecTests('unified-test-format/valid-fail'), () => false, true);
+});

test/spec/unified-test-format/valid-fail/entity-findCursor-malformed.json

@@ -0,0 +1,44 @@
+{
+  "description": "entity-findCursor-malformed",
+  "schemaVersion": "1.3",
+  "createEntities": [
+    {
+      "client": {
+        "id": "client0"
+      }
+    },
+    {
+      "database": {
+        "id": "database0",
+        "client": "client0",
+        "databaseName": "database0Name"
+      }
+    },
+    {
+      "collection": {
+        "id": "collection0",
+        "database": "database0",
+        "collectionName": "coll0"
+      }
+    }
+  ],
+  "initialData": [
+    {
+      "databaseName": "database0Name",
+      "collectionName": "coll0",
+      "documents": []
+    }
+  ],
+  "tests": [
+    {
+      "description": "createFindCursor fails if filter is not specified",
+      "operations": [
+        {
+          "name": "createFindCursor",
+          "object": "collection0",
+          "saveResultAsEntity": "cursor0"
+        }
+      ]
+    }
+  ]
+}

test/spec/unified-test-format/valid-fail/entity-findCursor-malformed.yml

@@ -0,0 +1,31 @@
+# This test is split out into a separate file to accommodate drivers that validate operation structure while decoding
+# from JSON/YML. Such drivers fail to decode any files containing invalid operations. Combining this test in a file
+# with other entity-findCursor valid-fail tests, which test failures that occur during test execution, would prevent
+# such drivers from decoding the file and running any of the tests.
+description: entity-findCursor-malformed
+
+schemaVersion: '1.3'
+
+createEntities:
+  - client:
+      id: &client0 client0
+  - database:
+      id: &database0 database0
+      client: *client0
+      databaseName: &database0Name database0Name
+  - collection:
+      id: &collection0 collection0
+      database: *database0
+      collectionName: &collection0Name coll0
+
+initialData:
+  - databaseName: *database0Name
+    collectionName: *collection0Name
+    documents: []
+
+tests:
+  - description: createFindCursor fails if filter is not specified
+    operations:
+      - name: createFindCursor
+        object: *collection0
+        saveResultAsEntity: &cursor0 cursor0

test/spec/unified-test-format/valid-fail/entity-findCursor.json

@@ -0,0 +1,52 @@
+{
+  "description": "entity-findCursor",
+  "schemaVersion": "1.3",
+  "createEntities": [
+    {
+      "client": {
+        "id": "client0"
+      }
+    },
+    {
+      "database": {
+        "id": "database0",
+        "client": "client0",
+        "databaseName": "database0Name"
+      }
+    },
+    {
+      "collection": {
+        "id": "collection0",
+        "database": "database0",
+        "collectionName": "coll0"
+      }
+    }
+  ],
+  "initialData": [
+    {
+      "databaseName": "database0Name",
+      "collectionName": "coll0",
+      "documents": []
+    }
+  ],
+  "tests": [
+    {
+      "description": "iterateUntilDocumentOrError fails if it references a nonexistent entity",
+      "operations": [
+        {
+          "name": "iterateUntilDocumentOrError",
+          "object": "cursor0"
+        }
+      ]
+    },
+    {
+      "description": "close fails if it references a nonexistent entity",
+      "operations": [
+        {
+          "name": "close",
+          "object": "cursor0"
+        }
+      ]
+    }
+  ]
+}

test/spec/unified-test-format/valid-fail/entity-findCursor.yml

@@ -0,0 +1,31 @@
+description: entity-findCursor
+
+schemaVersion: '1.3'
+
+createEntities:
+  - client:
+      id: &client0 client0
+  - database:
+      id: &database0 database0
+      client: *client0
+      databaseName: &database0Name database0Name
+  - collection:
+      id: &collection0 collection0
+      database: *database0
+      collectionName: &collection0Name coll0
+
+initialData:
+  - databaseName: *database0Name
+    collectionName: *collection0Name
+    documents: []
+
+tests:
+  - description: iterateUntilDocumentOrError fails if it references a nonexistent entity
+    operations:
+      - name: iterateUntilDocumentOrError
+        object: cursor0
+
+  - description: close fails if it references a nonexistent entity
+    operations:
+      - name: close
+        object: cursor0

test/spec/unified-test-format/valid-fail/ignoreResultAndError-malformed.json

@@ -0,0 +1,48 @@
+{
+  "description": "ignoreResultAndError-malformed",
+  "schemaVersion": "1.3",
+  "createEntities": [
+    {
+      "client": {
+        "id": "client0",
+        "useMultipleMongoses": true
+      }
+    },
+    {
+      "database": {
+        "id": "database0",
+        "client": "client0",
+        "databaseName": "database0Name"
+      }
+    },
+    {
+      "collection": {
+        "id": "collection0",
+        "database": "database0",
+        "collectionName": "coll0"
+      }
+    }
+  ],
+  "initialData": [
+    {
+      "collectionName": "coll0",
+      "databaseName": "database0Name",
+      "documents": []
+    }
+  ],
+  "tests": [
+    {
+      "description": "malformed operation fails if ignoreResultAndError is true",
+      "operations": [
+        {
+          "name": "insertOne",
+          "object": "collection0",
+          "arguments": {
+            "foo": "bar"
+          },
+          "ignoreResultAndError": true
+        }
+      ]
+    }
+  ]
+}

test/spec/unified-test-format/valid-fail/ignoreResultAndError-malformed.yml

@@ -0,0 +1,34 @@
+# This test is split out into a separate file to accommodate drivers that validate operation structure while decoding
+# from JSON/YML. Such drivers fail to decode any files containing invalid operations. Combining this test in a file
+# with other ignoreResultAndError valid-fail tests, which test failures that occur during test execution, would prevent
+# such drivers from decoding the file and running any of the tests.
+description: ignoreResultAndError-malformed  
+
+schemaVersion: '1.3'
+
+createEntities:
+  - client:
+      id: &client0 client0
+      useMultipleMongoses: true
+  - database:
+      id: &database0 database0
+      client: *client0
+      databaseName: &database0Name database0Name
+  - collection:
+      id: &collection0 collection0
+      database: *database0
+      collectionName: &collection0Name coll0
+
+initialData:
+  - collectionName: *collection0Name
+    databaseName: *database0Name
+    documents: []
+
+tests:
+  - description: malformed operation fails if ignoreResultAndError is true
+    operations:
+      - name: insertOne
+        object: *collection0
+        arguments:
+          foo: bar
+        ignoreResultAndError: true

test/spec/unified-test-format/valid-fail/ignoreResultAndError.json

@@ -54,19 +54,6 @@
           "ignoreResultAndError": false
         }
       ]
-    },
-    {
-      "description": "malformed operation fails if ignoreResultAndError is true",
-      "operations": [
-        {
-          "name": "insertOne",
-          "object": "collection0",
-          "arguments": {
-            "foo": "bar"
-          },
-          "ignoreResultAndError": true
-        }
-      ]
     }
   ]
 }

test/spec/unified-test-format/valid-fail/ignoreResultAndError.yml

@@ -33,11 +33,3 @@ tests:
           # Insert the same document to force a DuplicateKey error.
           document: *insertDocument
         ignoreResultAndError: false
-
-  - description: malformed operation fails if ignoreResultAndError is true
-    operations:
-      - name: insertOne
-        object: *collection0
-        arguments:
-          foo: bar
-        ignoreResultAndError: true

test/spec/unified-test-format/valid-fail/operation-unsupported.json

@@ -0,0 +1,22 @@
+{
+  "description": "operation-unsupported",
+  "schemaVersion": "1.0",
+  "createEntities": [
+    {
+      "client": {
+        "id": "client0"
+      }
+    }
+  ],
+  "tests": [
+    {
+      "description": "Unsupported operation",
+      "operations": [
+        {
+          "name": "unsupportedOperation",
+          "object": "client0"
+        }
+      ]
+    }
+  ]
+}

test/spec/unified-test-format/valid-fail/operation-unsupported.yml

@@ -0,0 +1,13 @@
+description: "operation-unsupported"
+
+schemaVersion: "1.0"
+
+createEntities:
+  - client:
+      id: &client0 client0
+
+tests:
+  - description: "Unsupported operation"
+    operations:
+      - name: unsupportedOperation
+        object: *client0