From 7f6322783c0922be3d120bc15b0d9ce9b689d2e5 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Fri, 30 Aug 2024 14:35:26 -0600 Subject: [PATCH 1/9] fix --- src/cmap/wire_protocol/on_demand/document.ts | 19 +- src/cmap/wire_protocol/responses.ts | 19 -- .../bson-options/utf8_validation.test.ts | 211 ++++++++++++++++++ 3 files changed, 227 insertions(+), 22 deletions(-) diff --git a/src/cmap/wire_protocol/on_demand/document.ts b/src/cmap/wire_protocol/on_demand/document.ts index 944916f10b..05b9d45dc4 100644 --- a/src/cmap/wire_protocol/on_demand/document.ts +++ b/src/cmap/wire_protocol/on_demand/document.ts @@ -10,6 +10,7 @@ import { getInt32LE, ObjectId, parseToElementsToArray, + pluckBSONSerializeOptions, Timestamp, toUTF8 } from '../../../bson'; @@ -330,11 +331,23 @@ export class OnDemandDocument { * @param options - BSON deserialization options */ public toObject(options?: BSONSerializeOptions): Record { - return BSON.deserialize(this.bson, { - ...options, + const exactBSONOptions = { + ...pluckBSONSerializeOptions(options ?? {}), + validation: this.parseBsonSerializationOptions(options), index: this.offset, allowObjectSmallerThanBufferSize: true - }); + }; + return BSON.deserialize(this.bson, exactBSONOptions); + } + + private parseBsonSerializationOptions(options?: { enableUtf8Validation?: boolean }): { + utf8: { writeErrors: false } | false; + } { + const enableUtf8Validation = options?.enableUtf8Validation; + if (enableUtf8Validation === false) { + return { utf8: false }; + } + return { utf8: { writeErrors: false } }; } /** Returns this document's bytes only */ diff --git a/src/cmap/wire_protocol/responses.ts b/src/cmap/wire_protocol/responses.ts index 0ef048e8da..9837634bfb 100644 --- a/src/cmap/wire_protocol/responses.ts +++ b/src/cmap/wire_protocol/responses.ts @@ -5,7 +5,6 @@ import { type Document, Long, parseToElementsToArray, - pluckBSONSerializeOptions, type Timestamp } from '../../bson'; import { MongoUnexpectedServerResponseError } from '../../error'; @@ -166,24 +165,6 @@ export class MongoDBResponse extends OnDemandDocument { } return this.clusterTime ?? null; } - - public override toObject(options?: BSONSerializeOptions): Record { - const exactBSONOptions = { - ...pluckBSONSerializeOptions(options ?? {}), - validation: this.parseBsonSerializationOptions(options) - }; - return super.toObject(exactBSONOptions); - } - - private parseBsonSerializationOptions(options?: { enableUtf8Validation?: boolean }): { - utf8: { writeErrors: false } | false; - } { - const enableUtf8Validation = options?.enableUtf8Validation; - if (enableUtf8Validation === false) { - return { utf8: false }; - } - return { utf8: { writeErrors: false } }; - } } /** @internal */ diff --git a/test/integration/node-specific/bson-options/utf8_validation.test.ts b/test/integration/node-specific/bson-options/utf8_validation.test.ts index 5c3f94e7fb..af391456cc 100644 --- a/test/integration/node-specific/bson-options/utf8_validation.test.ts +++ b/test/integration/node-specific/bson-options/utf8_validation.test.ts @@ -1,10 +1,14 @@ import { expect } from 'chai'; +import * as net from 'net'; import * as sinon from 'sinon'; import { BSON, + BSONError, + type Collection, type MongoClient, MongoDBResponse, + MongoError, MongoServerError, OpMsgResponse } from '../../../mongodb'; @@ -153,3 +157,210 @@ describe('class MongoDBResponse', () => { } ); }); + +describe('utf8 validation with cursors', function () { + let client: MongoClient; + let collection: Collection; + + /** + * Inserts a document with malformed utf8 bytes. This method spies on socket.write, and then waits + * for an OP_MSG payload corresponding to `collection.insertOne({ field: 'é' })`, and then modifies the + * bytes of the character 'é', to produce invalid utf8. + */ + async function insertDocumentWithInvalidUTF8() { + const targetCharacter = Buffer.from('é').toString('hex'); + + const stub = sinon.stub(net.Socket.prototype, 'write').callsFake(function (...args) { + const providedBuffer = args[0].toString('hex'); + const targetCharacter = Buffer.from('é').toString('hex'); + if (providedBuffer.includes(targetCharacter)) { + if (providedBuffer.split(targetCharacter).length !== 2) { + throw new Error('received buffer more than one `c3a9` sequences. or perhaps none?'); + } + const buffer = Buffer.from(providedBuffer.replace('c3a9', 'c301'), 'hex'); + const result = stub.wrappedMethod.apply(this, [buffer]); + sinon.restore(); + return result; + } + const result = stub.wrappedMethod.apply(this, args); + return result; + }); + + const document = { + field: targetCharacter + }; + + await collection.insertOne(document); + + sinon.restore(); + } + + beforeEach(async function () { + client = this.configuration.newClient(); + await client.connect(); + const db = client.db('test'); + collection = db.collection('invalidutf'); + + await collection.deleteMany({}); + await insertDocumentWithInvalidUTF8(); + }); + + afterEach(async function () { + await client.close(); + }); + + context('when utf-8 validation is explicitly disabled', function () { + it('documents can be read using a for-await loop without errors', async function () { + for await (const _doc of collection.find({}, { enableUtf8Validation: false })); + }); + it('documents can be read using next() without errors', async function () { + const cursor = collection.find({}, { enableUtf8Validation: false }); + + while (await cursor.hasNext()) { + await cursor.next(); + } + }); + + it('documents can be read using toArray() without errors', async function () { + const cursor = collection.find({}, { enableUtf8Validation: false }); + await cursor.toArray(); + }); + + it('documents can be read using .stream() without errors', async function () { + const cursor = collection.find({}, { enableUtf8Validation: false }); + await cursor.stream().toArray(); + }); + + it('documents can be read with tryNext() without error', async function () { + const cursor = collection.find({}, { enableUtf8Validation: false }); + + while (await cursor.hasNext()) { + await cursor.tryNext(); + } + }); + }); + + async function expectReject(fn: () => Promise, options?: { regex?: RegExp; errorClass }) { + const regex = options?.regex ?? /.*/; + const errorClass = options?.errorClass ?? MongoError; + try { + await fn(); + expect.fail('expected the provided callback function to reject, but it did not.'); + } catch (error) { + expect(error).to.match(regex); + expect(error).to.be.instanceOf(errorClass); + } + } + + context('when utf-8 validation is explicitly enabled', function () { + it('a for-await loop throw a BSON error', async function () { + await expectReject( + async () => { + for await (const _doc of collection.find({}, { enableUtf8Validation: true })); + }, + { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ } + ); + }); + it('next() throws a BSON error', async function () { + await expectReject( + async () => { + const cursor = collection.find({}, { enableUtf8Validation: true }); + + while (await cursor.hasNext()) { + await cursor.next(); + } + }, + { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ } + ); + }); + + it('toArray() throws a BSON error', async function () { + await expectReject( + async () => { + const cursor = collection.find({}, { enableUtf8Validation: true }); + await cursor.toArray(); + }, + { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ } + ); + }); + + it('.stream() throws a BSONError', async function () { + await expectReject( + async () => { + const cursor = collection.find({}, { enableUtf8Validation: true }); + await cursor.stream().toArray(); + }, + { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ } + ); + }); + + it('tryNext() throws a BSONError', async function () { + await expectReject( + async () => { + const cursor = collection.find({}, { enableUtf8Validation: true }); + + while (await cursor.hasNext()) { + await cursor.tryNext(); + } + }, + { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ } + ); + }); + }); + + context('utf-8 validation defaults to enabled', function () { + it('a for-await loop throw a BSON error', async function () { + await expectReject( + async () => { + for await (const _doc of collection.find({})); + }, + { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ } + ); + }); + it('next() throws a BSON error', async function () { + await expectReject( + async () => { + const cursor = collection.find({}); + + while (await cursor.hasNext()) { + await cursor.next(); + } + }, + { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ } + ); + }); + + it('toArray() throws a BSON error', async function () { + await expectReject( + async () => { + const cursor = collection.find({}); + await cursor.toArray(); + }, + { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ } + ); + }); + + it('.stream() throws a BSONError', async function () { + await expectReject( + async () => { + const cursor = collection.find({}); + await cursor.stream().toArray(); + }, + { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ } + ); + }); + + it('tryNext() throws a BSONError', async function () { + await expectReject( + async () => { + const cursor = collection.find({}, { enableUtf8Validation: true }); + + while (await cursor.hasNext()) { + await cursor.tryNext(); + } + }, + { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ } + ); + }); + }); +}); From 2b2110471972381f549bc0c31acf9f5dccea4296 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Fri, 30 Aug 2024 14:41:49 -0600 Subject: [PATCH 2/9] fix error --- .../bson-options/utf8_validation.test.ts | 139 +++++++----------- 1 file changed, 53 insertions(+), 86 deletions(-) diff --git a/test/integration/node-specific/bson-options/utf8_validation.test.ts b/test/integration/node-specific/bson-options/utf8_validation.test.ts index af391456cc..10469159b5 100644 --- a/test/integration/node-specific/bson-options/utf8_validation.test.ts +++ b/test/integration/node-specific/bson-options/utf8_validation.test.ts @@ -8,7 +8,6 @@ import { type Collection, type MongoClient, MongoDBResponse, - MongoError, MongoServerError, OpMsgResponse } from '../../../mongodb'; @@ -240,127 +239,95 @@ describe('utf8 validation with cursors', function () { }); }); - async function expectReject(fn: () => Promise, options?: { regex?: RegExp; errorClass }) { - const regex = options?.regex ?? /.*/; - const errorClass = options?.errorClass ?? MongoError; + async function expectReject(fn: () => Promise) { try { await fn(); expect.fail('expected the provided callback function to reject, but it did not.'); } catch (error) { - expect(error).to.match(regex); - expect(error).to.be.instanceOf(errorClass); + expect(error).to.match(/Invalid UTF-8 string in BSON document/); + expect(error).to.be.instanceOf(BSONError); } } context('when utf-8 validation is explicitly enabled', function () { it('a for-await loop throw a BSON error', async function () { - await expectReject( - async () => { - for await (const _doc of collection.find({}, { enableUtf8Validation: true })); - }, - { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ } - ); + await expectReject(async () => { + for await (const _doc of collection.find({}, { enableUtf8Validation: true })); + }); }); it('next() throws a BSON error', async function () { - await expectReject( - async () => { - const cursor = collection.find({}, { enableUtf8Validation: true }); - - while (await cursor.hasNext()) { - await cursor.next(); - } - }, - { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ } - ); + await expectReject(async () => { + const cursor = collection.find({}, { enableUtf8Validation: true }); + + while (await cursor.hasNext()) { + await cursor.next(); + } + }); }); it('toArray() throws a BSON error', async function () { - await expectReject( - async () => { - const cursor = collection.find({}, { enableUtf8Validation: true }); - await cursor.toArray(); - }, - { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ } - ); + await expectReject(async () => { + const cursor = collection.find({}, { enableUtf8Validation: true }); + await cursor.toArray(); + }); }); it('.stream() throws a BSONError', async function () { - await expectReject( - async () => { - const cursor = collection.find({}, { enableUtf8Validation: true }); - await cursor.stream().toArray(); - }, - { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ } - ); + await expectReject(async () => { + const cursor = collection.find({}, { enableUtf8Validation: true }); + await cursor.stream().toArray(); + }); }); it('tryNext() throws a BSONError', async function () { - await expectReject( - async () => { - const cursor = collection.find({}, { enableUtf8Validation: true }); - - while (await cursor.hasNext()) { - await cursor.tryNext(); - } - }, - { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ } - ); + await expectReject(async () => { + const cursor = collection.find({}, { enableUtf8Validation: true }); + + while (await cursor.hasNext()) { + await cursor.tryNext(); + } + }); }); }); context('utf-8 validation defaults to enabled', function () { it('a for-await loop throw a BSON error', async function () { - await expectReject( - async () => { - for await (const _doc of collection.find({})); - }, - { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ } - ); + await expectReject(async () => { + for await (const _doc of collection.find({})); + }); }); it('next() throws a BSON error', async function () { - await expectReject( - async () => { - const cursor = collection.find({}); - - while (await cursor.hasNext()) { - await cursor.next(); - } - }, - { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ } - ); + await expectReject(async () => { + const cursor = collection.find({}); + + while (await cursor.hasNext()) { + await cursor.next(); + } + }); }); it('toArray() throws a BSON error', async function () { - await expectReject( - async () => { - const cursor = collection.find({}); - await cursor.toArray(); - }, - { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ } - ); + await expectReject(async () => { + const cursor = collection.find({}); + await cursor.toArray(); + }); }); it('.stream() throws a BSONError', async function () { - await expectReject( - async () => { - const cursor = collection.find({}); - await cursor.stream().toArray(); - }, - { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ } - ); + await expectReject(async () => { + const cursor = collection.find({}); + await cursor.stream().toArray(); + }); }); it('tryNext() throws a BSONError', async function () { - await expectReject( - async () => { - const cursor = collection.find({}, { enableUtf8Validation: true }); - - while (await cursor.hasNext()) { - await cursor.tryNext(); - } - }, - { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ } - ); + await expectReject(async () => { + const cursor = collection.find({}, { enableUtf8Validation: true }); + + while (await cursor.hasNext()) { + await cursor.tryNext(); + } + }); }); }); }); From 62dd1045e335eeba80fa616c3455f517123e2b84 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Fri, 30 Aug 2024 15:01:07 -0600 Subject: [PATCH 3/9] fix tests --- .../bson-options/utf8_validation.test.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/test/integration/node-specific/bson-options/utf8_validation.test.ts b/test/integration/node-specific/bson-options/utf8_validation.test.ts index 10469159b5..2e873682e0 100644 --- a/test/integration/node-specific/bson-options/utf8_validation.test.ts +++ b/test/integration/node-specific/bson-options/utf8_validation.test.ts @@ -167,13 +167,12 @@ describe('utf8 validation with cursors', function () { * bytes of the character 'é', to produce invalid utf8. */ async function insertDocumentWithInvalidUTF8() { - const targetCharacter = Buffer.from('é').toString('hex'); - const stub = sinon.stub(net.Socket.prototype, 'write').callsFake(function (...args) { const providedBuffer = args[0].toString('hex'); - const targetCharacter = Buffer.from('é').toString('hex'); - if (providedBuffer.includes(targetCharacter)) { - if (providedBuffer.split(targetCharacter).length !== 2) { + const targetBytes = Buffer.from('é').toString('hex'); + + if (providedBuffer.includes(targetBytes)) { + if (providedBuffer.split(targetBytes).length !== 2) { throw new Error('received buffer more than one `c3a9` sequences. or perhaps none?'); } const buffer = Buffer.from(providedBuffer.replace('c3a9', 'c301'), 'hex'); @@ -186,7 +185,7 @@ describe('utf8 validation with cursors', function () { }); const document = { - field: targetCharacter + field: 'é' }; await collection.insertOne(document); From 45e19b65f8a62312b30a8a460ddaa83f4996a2e2 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Tue, 3 Sep 2024 08:28:59 -0600 Subject: [PATCH 4/9] comments --- .../node-specific/bson-options/utf8_validation.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/test/integration/node-specific/bson-options/utf8_validation.test.ts b/test/integration/node-specific/bson-options/utf8_validation.test.ts index 2e873682e0..3b14802d41 100644 --- a/test/integration/node-specific/bson-options/utf8_validation.test.ts +++ b/test/integration/node-specific/bson-options/utf8_validation.test.ts @@ -204,6 +204,7 @@ describe('utf8 validation with cursors', function () { }); afterEach(async function () { + sinon.restore(); await client.close(); }); From e90e3ff039f64398a8fe78235292bf6df91daf5a Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Tue, 3 Sep 2024 10:49:37 -0600 Subject: [PATCH 5/9] fix unit tests --- src/cmap/wire_protocol/on_demand/document.ts | 9 ++++- .../unit/cmap/wire_protocol/responses.test.ts | 40 ++++++++++++++----- 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/src/cmap/wire_protocol/on_demand/document.ts b/src/cmap/wire_protocol/on_demand/document.ts index 05b9d45dc4..5034053030 100644 --- a/src/cmap/wire_protocol/on_demand/document.ts +++ b/src/cmap/wire_protocol/on_demand/document.ts @@ -46,6 +46,13 @@ type CachedBSONElement = { element: BSONElement; value: any | undefined }; /** @internal */ export class OnDemandDocument { + /** + * @internal + * + * Used for testing purposes. + */ + private static BSON: typeof BSON = BSON; + /** * Maps JS strings to elements and jsValues for speeding up subsequent lookups. * - If `false` then name does not exist in the BSON document @@ -337,7 +344,7 @@ export class OnDemandDocument { index: this.offset, allowObjectSmallerThanBufferSize: true }; - return BSON.deserialize(this.bson, exactBSONOptions); + return OnDemandDocument.BSON.deserialize(this.bson, exactBSONOptions); } private parseBsonSerializationOptions(options?: { enableUtf8Validation?: boolean }): { diff --git a/test/unit/cmap/wire_protocol/responses.test.ts b/test/unit/cmap/wire_protocol/responses.test.ts index 9498765cf4..5e0fd6e993 100644 --- a/test/unit/cmap/wire_protocol/responses.test.ts +++ b/test/unit/cmap/wire_protocol/responses.test.ts @@ -1,3 +1,4 @@ +import * as SPYABLE_BSON from 'bson'; import { expect } from 'chai'; import * as sinon from 'sinon'; @@ -16,17 +17,28 @@ describe('class MongoDBResponse', () => { }); context('utf8 validation', () => { - afterEach(() => sinon.restore()); + let deseriailzeSpy: sinon.SinonSpy; + beforeEach(function () { + // @ts-expect-error accessing internal property. + OnDemandDocument.BSON = SPYABLE_BSON; + + deseriailzeSpy = sinon.spy(SPYABLE_BSON, 'deserialize'); + }); + afterEach(function () { + sinon.restore(); + }); context('when enableUtf8Validation is not specified', () => { const options = { enableUtf8Validation: undefined }; it('calls BSON deserialize with writeErrors validation turned off', () => { const res = new MongoDBResponse(BSON.serialize({})); - const toObject = sinon.spy(Object.getPrototypeOf(Object.getPrototypeOf(res)), 'toObject'); res.toObject(options); - expect(toObject).to.have.been.calledWith( - sinon.match({ validation: { utf8: { writeErrors: false } } }) - ); + + expect(deseriailzeSpy).to.have.been.called; + + const [_buffer, { validation }] = deseriailzeSpy.getCalls()[0].args; + + expect(validation).to.deep.equal({ utf8: { writeErrors: false } }); }); }); @@ -34,11 +46,13 @@ describe('class MongoDBResponse', () => { const options = { enableUtf8Validation: true }; it('calls BSON deserialize with writeErrors validation turned off', () => { const res = new MongoDBResponse(BSON.serialize({})); - const toObject = sinon.spy(Object.getPrototypeOf(Object.getPrototypeOf(res)), 'toObject'); res.toObject(options); - expect(toObject).to.have.been.calledWith( - sinon.match({ validation: { utf8: { writeErrors: false } } }) - ); + + expect(deseriailzeSpy).to.have.been.called; + + const [_buffer, { validation }] = deseriailzeSpy.getCalls()[0].args; + + expect(validation).to.deep.equal({ utf8: { writeErrors: false } }); }); }); @@ -46,9 +60,13 @@ describe('class MongoDBResponse', () => { const options = { enableUtf8Validation: false }; it('calls BSON deserialize with all validation disabled', () => { const res = new MongoDBResponse(BSON.serialize({})); - const toObject = sinon.spy(Object.getPrototypeOf(Object.getPrototypeOf(res)), 'toObject'); res.toObject(options); - expect(toObject).to.have.been.calledWith(sinon.match({ validation: { utf8: false } })); + + expect(deseriailzeSpy).to.have.been.called; + + const [_buffer, { validation }] = deseriailzeSpy.getCalls()[0].args; + + expect(validation).to.deep.equal({ utf8: false }); }); }); }); From aec297a303baa4300a5ac55947a19970c08f135e Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Tue, 3 Sep 2024 10:52:35 -0600 Subject: [PATCH 6/9] little bit of code golf --- test/unit/cmap/wire_protocol/responses.test.ts | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/test/unit/cmap/wire_protocol/responses.test.ts b/test/unit/cmap/wire_protocol/responses.test.ts index 5e0fd6e993..78d6a90ae2 100644 --- a/test/unit/cmap/wire_protocol/responses.test.ts +++ b/test/unit/cmap/wire_protocol/responses.test.ts @@ -36,7 +36,11 @@ describe('class MongoDBResponse', () => { expect(deseriailzeSpy).to.have.been.called; - const [_buffer, { validation }] = deseriailzeSpy.getCalls()[0].args; + const [ + { + args: [_buffer, { validation }] + } + ] = deseriailzeSpy.getCalls(); expect(validation).to.deep.equal({ utf8: { writeErrors: false } }); }); @@ -50,7 +54,11 @@ describe('class MongoDBResponse', () => { expect(deseriailzeSpy).to.have.been.called; - const [_buffer, { validation }] = deseriailzeSpy.getCalls()[0].args; + const [ + { + args: [_buffer, { validation }] + } + ] = deseriailzeSpy.getCalls(); expect(validation).to.deep.equal({ utf8: { writeErrors: false } }); }); @@ -64,7 +72,11 @@ describe('class MongoDBResponse', () => { expect(deseriailzeSpy).to.have.been.called; - const [_buffer, { validation }] = deseriailzeSpy.getCalls()[0].args; + const [ + { + args: [_buffer, { validation }] + } + ] = deseriailzeSpy.getCalls(); expect(validation).to.deep.equal({ utf8: false }); }); From 971a65cb9349ac175361157711eabd71464fd8dc Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Tue, 3 Sep 2024 19:13:02 -0600 Subject: [PATCH 7/9] add debug information --- .evergreen/install-mongodb-client-encryption.sh | 2 +- .../node-specific/bson-options/utf8_validation.test.ts | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.evergreen/install-mongodb-client-encryption.sh b/.evergreen/install-mongodb-client-encryption.sh index 80432960f3..f8ab200869 100644 --- a/.evergreen/install-mongodb-client-encryption.sh +++ b/.evergreen/install-mongodb-client-encryption.sh @@ -10,7 +10,7 @@ set -o xtrace # Write all commands first to stderr set -o errexit # Exit the script with error if any of the commands fail rm -rf mongodb-client-encryption -git clone https://github.com/mongodb-js/mongodb-client-encryption.git +git clone https://github.com/mongodb-js/mongodb-client-encryption.git -b explicit-lifetime-chaining pushd mongodb-client-encryption if [ -n "${LIBMONGOCRYPT_VERSION}" ]; then diff --git a/test/integration/node-specific/bson-options/utf8_validation.test.ts b/test/integration/node-specific/bson-options/utf8_validation.test.ts index 3b14802d41..176ade8d11 100644 --- a/test/integration/node-specific/bson-options/utf8_validation.test.ts +++ b/test/integration/node-specific/bson-options/utf8_validation.test.ts @@ -1,11 +1,13 @@ import { expect } from 'chai'; import * as net from 'net'; import * as sinon from 'sinon'; +import { inspect } from 'util'; import { BSON, BSONError, type Collection, + deserialize, type MongoClient, MongoDBResponse, MongoServerError, @@ -173,7 +175,9 @@ describe('utf8 validation with cursors', function () { if (providedBuffer.includes(targetBytes)) { if (providedBuffer.split(targetBytes).length !== 2) { - throw new Error('received buffer more than one `c3a9` sequences. or perhaps none?'); + sinon.restore(); + const message = `expected exactly one c3a9 sequence, received ${providedBuffer.split(targetBytes).length}\n. command: ${inspect(deserialize(args[0]), { depth: Infinity })}`; + throw new Error(message); } const buffer = Buffer.from(providedBuffer.replace('c3a9', 'c301'), 'hex'); const result = stub.wrappedMethod.apply(this, [buffer]); From 46e779da02d5bf96e8449f9385ed17b5c2c95816 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Wed, 4 Sep 2024 07:29:56 -0600 Subject: [PATCH 8/9] make byte sequence more unique --- .evergreen/install-mongodb-client-encryption.sh | 2 +- .../node-specific/bson-options/utf8_validation.test.ts | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.evergreen/install-mongodb-client-encryption.sh b/.evergreen/install-mongodb-client-encryption.sh index f8ab200869..80432960f3 100644 --- a/.evergreen/install-mongodb-client-encryption.sh +++ b/.evergreen/install-mongodb-client-encryption.sh @@ -10,7 +10,7 @@ set -o xtrace # Write all commands first to stderr set -o errexit # Exit the script with error if any of the commands fail rm -rf mongodb-client-encryption -git clone https://github.com/mongodb-js/mongodb-client-encryption.git -b explicit-lifetime-chaining +git clone https://github.com/mongodb-js/mongodb-client-encryption.git pushd mongodb-client-encryption if [ -n "${LIBMONGOCRYPT_VERSION}" ]; then diff --git a/test/integration/node-specific/bson-options/utf8_validation.test.ts b/test/integration/node-specific/bson-options/utf8_validation.test.ts index 176ade8d11..675e0af39e 100644 --- a/test/integration/node-specific/bson-options/utf8_validation.test.ts +++ b/test/integration/node-specific/bson-options/utf8_validation.test.ts @@ -159,7 +159,7 @@ describe('class MongoDBResponse', () => { ); }); -describe('utf8 validation with cursors', function () { +describe('utf8 validation with cursors' + i, function () { let client: MongoClient; let collection: Collection; @@ -171,15 +171,15 @@ describe('utf8 validation with cursors', function () { async function insertDocumentWithInvalidUTF8() { const stub = sinon.stub(net.Socket.prototype, 'write').callsFake(function (...args) { const providedBuffer = args[0].toString('hex'); - const targetBytes = Buffer.from('é').toString('hex'); + const targetBytes = Buffer.from(document.field, 'utf-8').toString('hex'); if (providedBuffer.includes(targetBytes)) { if (providedBuffer.split(targetBytes).length !== 2) { sinon.restore(); - const message = `expected exactly one c3a9 sequence, received ${providedBuffer.split(targetBytes).length}\n. command: ${inspect(deserialize(args[0]), { depth: Infinity })}`; + const message = `too many target bytes sequences: received ${providedBuffer.split(targetBytes).length}\n. command: ${inspect(deserialize(args[0]), { depth: Infinity })}`; throw new Error(message); } - const buffer = Buffer.from(providedBuffer.replace('c3a9', 'c301'), 'hex'); + const buffer = Buffer.from(providedBuffer.replace(targetBytes, 'c301'.repeat(8)), 'hex'); const result = stub.wrappedMethod.apply(this, [buffer]); sinon.restore(); return result; @@ -189,7 +189,7 @@ describe('utf8 validation with cursors', function () { }); const document = { - field: 'é' + field: 'é'.repeat(8) }; await collection.insertOne(document); From c169e585c8bd3651310142dc2b03b6c465ad584d Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Wed, 4 Sep 2024 08:21:51 -0600 Subject: [PATCH 9/9] spy instead of injecting --- src/cmap/wire_protocol/on_demand/document.ts | 11 +---- .../bson-options/utf8_validation.test.ts | 8 ++-- .../unit/cmap/wire_protocol/responses.test.ts | 45 ++++++++++--------- 3 files changed, 30 insertions(+), 34 deletions(-) diff --git a/src/cmap/wire_protocol/on_demand/document.ts b/src/cmap/wire_protocol/on_demand/document.ts index 5034053030..67f5b3a091 100644 --- a/src/cmap/wire_protocol/on_demand/document.ts +++ b/src/cmap/wire_protocol/on_demand/document.ts @@ -1,10 +1,10 @@ import { Binary, - BSON, type BSONElement, BSONError, type BSONSerializeOptions, BSONType, + deserialize, getBigInt64LE, getFloat64LE, getInt32LE, @@ -46,13 +46,6 @@ type CachedBSONElement = { element: BSONElement; value: any | undefined }; /** @internal */ export class OnDemandDocument { - /** - * @internal - * - * Used for testing purposes. - */ - private static BSON: typeof BSON = BSON; - /** * Maps JS strings to elements and jsValues for speeding up subsequent lookups. * - If `false` then name does not exist in the BSON document @@ -344,7 +337,7 @@ export class OnDemandDocument { index: this.offset, allowObjectSmallerThanBufferSize: true }; - return OnDemandDocument.BSON.deserialize(this.bson, exactBSONOptions); + return deserialize(this.bson, exactBSONOptions); } private parseBsonSerializationOptions(options?: { enableUtf8Validation?: boolean }): { diff --git a/test/integration/node-specific/bson-options/utf8_validation.test.ts b/test/integration/node-specific/bson-options/utf8_validation.test.ts index 675e0af39e..d6345a884d 100644 --- a/test/integration/node-specific/bson-options/utf8_validation.test.ts +++ b/test/integration/node-specific/bson-options/utf8_validation.test.ts @@ -9,8 +9,8 @@ import { type Collection, deserialize, type MongoClient, - MongoDBResponse, MongoServerError, + OnDemandDocument, OpMsgResponse } from '../../../mongodb'; @@ -28,12 +28,12 @@ describe('class MongoDBResponse', () => { let bsonSpy: sinon.SinonSpy; beforeEach(() => { - bsonSpy = sinon.spy(MongoDBResponse.prototype, 'parseBsonSerializationOptions'); + // @ts-expect-error private function + bsonSpy = sinon.spy(OnDemandDocument.prototype, 'parseBsonSerializationOptions'); }); afterEach(() => { bsonSpy?.restore(); - // @ts-expect-error: Allow this to be garbage collected bsonSpy = null; }); @@ -159,7 +159,7 @@ describe('class MongoDBResponse', () => { ); }); -describe('utf8 validation with cursors' + i, function () { +describe('utf8 validation with cursors', function () { let client: MongoClient; let collection: Collection; diff --git a/test/unit/cmap/wire_protocol/responses.test.ts b/test/unit/cmap/wire_protocol/responses.test.ts index 78d6a90ae2..7fccbfc7fc 100644 --- a/test/unit/cmap/wire_protocol/responses.test.ts +++ b/test/unit/cmap/wire_protocol/responses.test.ts @@ -1,28 +1,31 @@ -import * as SPYABLE_BSON from 'bson'; import { expect } from 'chai'; import * as sinon from 'sinon'; +// to spy on the bson module, we must import it from the driver +// eslint-disable-next-line @typescript-eslint/no-restricted-imports +import * as mdb from '../../../../src/bson'; import { - BSON, CursorResponse, Int32, MongoDBResponse, MongoUnexpectedServerResponseError, - OnDemandDocument + OnDemandDocument, + serialize } from '../../../mongodb'; describe('class MongoDBResponse', () => { it('is a subclass of OnDemandDocument', () => { - expect(new MongoDBResponse(BSON.serialize({ ok: 1 }))).to.be.instanceOf(OnDemandDocument); + expect(new MongoDBResponse(serialize({ ok: 1 }))).to.be.instanceOf(OnDemandDocument); }); context('utf8 validation', () => { - let deseriailzeSpy: sinon.SinonSpy; + let deseriailzeSpy: sinon.SinonStub>; beforeEach(function () { - // @ts-expect-error accessing internal property. - OnDemandDocument.BSON = SPYABLE_BSON; - - deseriailzeSpy = sinon.spy(SPYABLE_BSON, 'deserialize'); + const deserialize = mdb.deserialize; + deseriailzeSpy = sinon.stub>().callsFake(deserialize); + sinon.stub(mdb, 'deserialize').get(() => { + return deseriailzeSpy; + }); }); afterEach(function () { sinon.restore(); @@ -31,7 +34,7 @@ describe('class MongoDBResponse', () => { context('when enableUtf8Validation is not specified', () => { const options = { enableUtf8Validation: undefined }; it('calls BSON deserialize with writeErrors validation turned off', () => { - const res = new MongoDBResponse(BSON.serialize({})); + const res = new MongoDBResponse(serialize({})); res.toObject(options); expect(deseriailzeSpy).to.have.been.called; @@ -49,7 +52,7 @@ describe('class MongoDBResponse', () => { context('when enableUtf8Validation is true', () => { const options = { enableUtf8Validation: true }; it('calls BSON deserialize with writeErrors validation turned off', () => { - const res = new MongoDBResponse(BSON.serialize({})); + const res = new MongoDBResponse(serialize({})); res.toObject(options); expect(deseriailzeSpy).to.have.been.called; @@ -67,7 +70,7 @@ describe('class MongoDBResponse', () => { context('when enableUtf8Validation is false', () => { const options = { enableUtf8Validation: false }; it('calls BSON deserialize with all validation disabled', () => { - const res = new MongoDBResponse(BSON.serialize({})); + const res = new MongoDBResponse(serialize({})); res.toObject(options); expect(deseriailzeSpy).to.have.been.called; @@ -87,7 +90,7 @@ describe('class MongoDBResponse', () => { describe('class CursorResponse', () => { describe('get cursor()', () => { it('throws if input does not contain cursor embedded document', () => { - expect(() => new CursorResponse(BSON.serialize({ ok: 1 })).cursor).to.throw( + expect(() => new CursorResponse(serialize({ ok: 1 })).cursor).to.throw( MongoUnexpectedServerResponseError, /"cursor" is missing/ ); @@ -96,7 +99,7 @@ describe('class CursorResponse', () => { describe('get id()', () => { it('throws if input does not contain cursor.id int64', () => { - expect(() => new CursorResponse(BSON.serialize({ ok: 1, cursor: {} })).id).to.throw( + expect(() => new CursorResponse(serialize({ ok: 1, cursor: {} })).id).to.throw( MongoUnexpectedServerResponseError, /"id" is missing/ ); @@ -107,22 +110,22 @@ describe('class CursorResponse', () => { it('throws if input does not contain firstBatch nor nextBatch', () => { expect( // @ts-expect-error: testing private getter - () => new CursorResponse(BSON.serialize({ ok: 1, cursor: { id: 0n, batch: [] } })).batch + () => new CursorResponse(serialize({ ok: 1, cursor: { id: 0n, batch: [] } })).batch ).to.throw(MongoUnexpectedServerResponseError, /did not contain a batch/); }); }); describe('get ns()', () => { it('sets namespace to null if input does not contain cursor.ns', () => { - expect(new CursorResponse(BSON.serialize({ ok: 1, cursor: { id: 0n, firstBatch: [] } })).ns) - .to.be.null; + expect(new CursorResponse(serialize({ ok: 1, cursor: { id: 0n, firstBatch: [] } })).ns).to.be + .null; }); }); describe('get batchSize()', () => { it('reports the returned batch size', () => { const response = new CursorResponse( - BSON.serialize({ ok: 1, cursor: { id: 0n, nextBatch: [{}, {}, {}] } }) + serialize({ ok: 1, cursor: { id: 0n, nextBatch: [{}, {}, {}] } }) ); expect(response.batchSize).to.equal(3); expect(response.shift()).to.deep.equal({}); @@ -133,7 +136,7 @@ describe('class CursorResponse', () => { describe('get length()', () => { it('reports number of documents remaining in the batch', () => { const response = new CursorResponse( - BSON.serialize({ ok: 1, cursor: { id: 0n, nextBatch: [{}, {}, {}] } }) + serialize({ ok: 1, cursor: { id: 0n, nextBatch: [{}, {}, {}] } }) ); expect(response).to.have.lengthOf(3); expect(response.shift()).to.deep.equal({}); @@ -146,7 +149,7 @@ describe('class CursorResponse', () => { beforeEach(async function () { response = new CursorResponse( - BSON.serialize({ + serialize({ ok: 1, cursor: { id: 0n, nextBatch: [{ _id: 1 }, { _id: 2 }, { _id: 3 }] } }) @@ -173,7 +176,7 @@ describe('class CursorResponse', () => { beforeEach(async function () { response = new CursorResponse( - BSON.serialize({ + serialize({ ok: 1, cursor: { id: 0n, nextBatch: [{ _id: 1 }, { _id: 2 }, { _id: 3 }] } })