From 1c0891a75c4b42ee1bcc6976f1a2cce216bd2895 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Fri, 6 Sep 2024 10:32:08 -0600 Subject: [PATCH] asdf --- src/bson.ts | 11 ++ src/cmap/connection.ts | 3 +- src/cmap/wire_protocol/on_demand/document.ts | 23 ++- src/cmap/wire_protocol/responses.ts | 28 +++- src/cursor/abstract_cursor.ts | 28 +++- src/cursor/aggregation_cursor.ts | 2 +- src/cursor/find_cursor.ts | 2 +- src/index.ts | 6 +- .../bson-options/utf8_validation.test.ts | 141 ++++-------------- .../unit/cmap/wire_protocol/responses.test.ts | 72 --------- 10 files changed, 106 insertions(+), 210 deletions(-) diff --git a/src/bson.ts b/src/bson.ts index ce5bd486b7..d749db2c4a 100644 --- a/src/bson.ts +++ b/src/bson.ts @@ -133,3 +133,14 @@ export function resolveBSONOptions( options?.enableUtf8Validation ?? parentOptions?.enableUtf8Validation ?? true }; } + +/** @internal */ +export function parseUtf8ValidationOption(options?: { enableUtf8Validation?: boolean }): { + utf8: { writeErrors: false } | false; +} { + const enableUtf8Validation = options?.enableUtf8Validation; + if (enableUtf8Validation === false) { + return { utf8: false }; + } + return { utf8: { writeErrors: false } }; +} diff --git a/src/cmap/connection.ts b/src/cmap/connection.ts index 1e4afc7f38..dd823ee07e 100644 --- a/src/cmap/connection.ts +++ b/src/cmap/connection.ts @@ -1,3 +1,4 @@ +import { type DeserializeOptions } from 'bson'; import { type Readable, Transform, type TransformCallback } from 'stream'; import { clearTimeout, setTimeout } from 'timers'; @@ -487,7 +488,7 @@ export class Connection extends TypedEventEmitter { // If `documentsReturnedIn` not set or raw is not enabled, use input bson options // Otherwise, support raw flag. Raw only works for cursors that hardcode firstBatch/nextBatch fields - const bsonOptions = + const bsonOptions: DeserializeOptions = options.documentsReturnedIn == null || !options.raw ? options : { diff --git a/src/cmap/wire_protocol/on_demand/document.ts b/src/cmap/wire_protocol/on_demand/document.ts index 67f5b3a091..fdb6f90e4d 100644 --- a/src/cmap/wire_protocol/on_demand/document.ts +++ b/src/cmap/wire_protocol/on_demand/document.ts @@ -1,8 +1,9 @@ +import { type DeserializeOptions } from 'bson'; + import { Binary, type BSONElement, BSONError, - type BSONSerializeOptions, BSONType, deserialize, getBigInt64LE, @@ -10,7 +11,6 @@ import { getInt32LE, ObjectId, parseToElementsToArray, - pluckBSONSerializeOptions, Timestamp, toUTF8 } from '../../../bson'; @@ -44,6 +44,15 @@ export type JSTypeOf = { /** @internal */ type CachedBSONElement = { element: BSONElement; value: any | undefined }; +/** + * @internal + * + * Options for `OnDemandDocument.toObject()`. Validation is required to ensure + * that callers provide utf8 validation options. */ +export type OnDemandDocumentDeserializeOptions = DeserializeOptions & { + validation: NonNullable; +}; + /** @internal */ export class OnDemandDocument { /** @@ -330,14 +339,12 @@ export class OnDemandDocument { * Deserialize this object, DOES NOT cache result so avoid multiple invocations * @param options - BSON deserialization options */ - public toObject(options?: BSONSerializeOptions): Record { - const exactBSONOptions = { - ...pluckBSONSerializeOptions(options ?? {}), - validation: this.parseBsonSerializationOptions(options), + public toObject(options?: OnDemandDocumentDeserializeOptions): Record { + return deserialize(this.bson, { + ...options, index: this.offset, allowObjectSmallerThanBufferSize: true - }; - return deserialize(this.bson, exactBSONOptions); + }); } private parseBsonSerializationOptions(options?: { enableUtf8Validation?: boolean }): { diff --git a/src/cmap/wire_protocol/responses.ts b/src/cmap/wire_protocol/responses.ts index 9837634bfb..e69cf84cfc 100644 --- a/src/cmap/wire_protocol/responses.ts +++ b/src/cmap/wire_protocol/responses.ts @@ -1,3 +1,5 @@ +import { type DeserializeOptions } from 'bson'; + import { type BSONElement, type BSONSerializeOptions, @@ -5,12 +7,18 @@ import { type Document, Long, parseToElementsToArray, + parseUtf8ValidationOption, + pluckBSONSerializeOptions, type Timestamp } from '../../bson'; import { MongoUnexpectedServerResponseError } from '../../error'; import { type ClusterTime } from '../../sdam/common'; import { decorateDecryptionResult, ns } from '../../utils'; -import { type JSTypeOf, OnDemandDocument } from './on_demand/document'; +import { + type JSTypeOf, + OnDemandDocument, + type OnDemandDocumentDeserializeOptions +} from './on_demand/document'; // eslint-disable-next-line no-restricted-syntax const enum BSONElementOffset { @@ -112,7 +120,8 @@ export class MongoDBResponse extends OnDemandDocument { this.get('recoveryToken', BSONType.object)?.toObject({ promoteValues: false, promoteLongs: false, - promoteBuffers: false + promoteBuffers: false, + validation: { utf8: true } }) ?? null ); } @@ -165,6 +174,14 @@ export class MongoDBResponse extends OnDemandDocument { } return this.clusterTime ?? null; } + + public override toObject(options?: BSONSerializeOptions): Record { + const exactBSONOptions = { + ...pluckBSONSerializeOptions(options ?? {}), + validation: parseUtf8ValidationOption(options) + }; + return super.toObject(exactBSONOptions); + } } /** @internal */ @@ -248,12 +265,13 @@ export class CursorResponse extends MongoDBResponse { this.cursor.get('postBatchResumeToken', BSONType.object)?.toObject({ promoteValues: false, promoteLongs: false, - promoteBuffers: false + promoteBuffers: false, + validation: { utf8: true } }) ?? null ); } - public shift(options?: BSONSerializeOptions): any { + public shift(options: OnDemandDocumentDeserializeOptions): any { if (this.iterated >= this.batchSize) { return null; } @@ -305,7 +323,7 @@ export class ExplainedCursorResponse extends CursorResponse { return this._length; } - override shift(options?: BSONSerializeOptions | undefined) { + override shift(options?: DeserializeOptions) { if (this._length === 0) return null; this._length -= 1; return this.toObject(options); diff --git a/src/cursor/abstract_cursor.ts b/src/cursor/abstract_cursor.ts index da08f1a1a6..08a8bf7b31 100644 --- a/src/cursor/abstract_cursor.ts +++ b/src/cursor/abstract_cursor.ts @@ -1,6 +1,14 @@ +import { type DeserializeOptions } from 'bson'; import { Readable, Transform } from 'stream'; -import { type BSONSerializeOptions, type Document, Long, pluckBSONSerializeOptions } from '../bson'; +import { + type BSONSerializeOptions, + type Document, + Long, + parseUtf8ValidationOption, + pluckBSONSerializeOptions +} from '../bson'; +import { type OnDemandDocumentDeserializeOptions } from '../cmap/wire_protocol/on_demand/document'; import { type CursorResponse } from '../cmap/wire_protocol/responses'; import { MongoAPIError, @@ -153,6 +161,9 @@ export abstract class AbstractCursor< /** @event */ static readonly CLOSE = 'close' as const; + /** @internal */ + protected deserializationOptions: OnDemandDocumentDeserializeOptions; + /** @internal */ protected constructor( client: MongoClient, @@ -207,6 +218,13 @@ export abstract class AbstractCursor< } else { this.cursorSession = this.cursorClient.startSession({ owner: this, explicit: false }); } + + this.deserializationOptions = { + ...this.cursorOptions, + validation: { + utf8: options?.enableUtf8Validation === false ? false : true + } + }; } /** @@ -289,7 +307,7 @@ export abstract class AbstractCursor< ); for (let count = 0; count < documentsToRead; count++) { - const document = this.documents?.shift(this.cursorOptions); + const document = this.documents?.shift(this.deserializationOptions); if (document != null) { bufferedDocs.push(document); } @@ -390,7 +408,7 @@ export abstract class AbstractCursor< } do { - const doc = this.documents?.shift(this.cursorOptions); + const doc = this.documents?.shift(this.deserializationOptions); if (doc != null) { if (this.transform != null) return await this.transformDocument(doc); return doc; @@ -409,7 +427,7 @@ export abstract class AbstractCursor< throw new MongoCursorExhaustedError(); } - let doc = this.documents?.shift(this.cursorOptions); + let doc = this.documents?.shift(this.deserializationOptions); if (doc != null) { if (this.transform != null) return await this.transformDocument(doc); return doc; @@ -417,7 +435,7 @@ export abstract class AbstractCursor< await this.fetchBatch(); - doc = this.documents?.shift(this.cursorOptions); + doc = this.documents?.shift(this.deserializationOptions); if (doc != null) { if (this.transform != null) return await this.transformDocument(doc); return doc; diff --git a/src/cursor/aggregation_cursor.ts b/src/cursor/aggregation_cursor.ts index 1b1ad663ba..3281e77707 100644 --- a/src/cursor/aggregation_cursor.ts +++ b/src/cursor/aggregation_cursor.ts @@ -74,7 +74,7 @@ export class AggregationCursor extends AbstractCursor { explain: verbosity ?? true }) ) - ).shift(this.aggregateOptions); + ).shift(this.deserializationOptions); } /** Add a stage to the aggregation pipeline diff --git a/src/cursor/find_cursor.ts b/src/cursor/find_cursor.ts index e4b3dbc03c..ef21cea290 100644 --- a/src/cursor/find_cursor.ts +++ b/src/cursor/find_cursor.ts @@ -143,7 +143,7 @@ export class FindCursor extends AbstractCursor { explain: verbosity ?? true }) ) - ).shift(this.findOptions); + ).shift(this.deserializationOptions); } /** Set the cursor query */ diff --git a/src/index.ts b/src/index.ts index 8bf6c68617..6d0908198f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -299,7 +299,11 @@ export type { ClientMetadata, ClientMetadataOptions } from './cmap/handshake/cli export type { ConnectionPoolMetrics } from './cmap/metrics'; export type { StreamDescription, StreamDescriptionOptions } from './cmap/stream_description'; export type { CompressorName } from './cmap/wire_protocol/compression'; -export type { JSTypeOf, OnDemandDocument } from './cmap/wire_protocol/on_demand/document'; +export type { + JSTypeOf, + OnDemandDocument, + OnDemandDocumentDeserializeOptions +} from './cmap/wire_protocol/on_demand/document'; export type { CursorResponse, MongoDBResponse, diff --git a/test/integration/node-specific/bson-options/utf8_validation.test.ts b/test/integration/node-specific/bson-options/utf8_validation.test.ts index d6345a884d..df367f233b 100644 --- a/test/integration/node-specific/bson-options/utf8_validation.test.ts +++ b/test/integration/node-specific/bson-options/utf8_validation.test.ts @@ -10,121 +10,43 @@ import { deserialize, type MongoClient, MongoServerError, - OnDemandDocument, OpMsgResponse } from '../../../mongodb'; -const EXPECTED_VALIDATION_DISABLED_ARGUMENT = { - utf8: false -}; - -const EXPECTED_VALIDATION_ENABLED_ARGUMENT = { - utf8: { - writeErrors: false - } -}; - describe('class MongoDBResponse', () => { - let bsonSpy: sinon.SinonSpy; - - beforeEach(() => { - // @ts-expect-error private function - bsonSpy = sinon.spy(OnDemandDocument.prototype, 'parseBsonSerializationOptions'); - }); - - afterEach(() => { - bsonSpy?.restore(); - bsonSpy = null; - }); - let client; afterEach(async () => { + sinon.restore(); if (client) await client.close(); }); - describe('enableUtf8Validation option set to false', () => { - const option = { enableUtf8Validation: false }; + context( + 'when the server is given a long multibyte utf sequence and there is a writeError that includes invalid utf8', + () => { + let client: MongoClient; + let error: MongoServerError; + beforeEach(async function () { + client = this.configuration.newClient(); - for (const passOptionTo of ['client', 'db', 'collection', 'operation']) { - it(`should disable validation with option passed to ${passOptionTo}`, async function () { - client = this.configuration.newClient(passOptionTo === 'client' ? option : undefined); + async function generateWriteErrorWithInvalidUtf8() { + // Insert a large string of multibyte UTF-8 characters + const _id = '\u{1F92A}'.repeat(100); - const db = client.db('bson_utf8Validation_db', passOptionTo === 'db' ? option : undefined); - const collection = db.collection( - 'bson_utf8Validation_coll', - passOptionTo === 'collection' ? option : undefined - ); + const test = client.db('parsing').collection<{ _id: string }>('parsing'); + await test.insertOne({ _id }); - await collection.insertOne( - { name: 'John Doe' }, - passOptionTo === 'operation' ? option : {} - ); + const spy = sinon.spy(OpMsgResponse.prototype, 'parse'); - expect(bsonSpy).to.have.been.called; - const result = bsonSpy.lastCall.returnValue; - expect(result).to.deep.equal(EXPECTED_VALIDATION_DISABLED_ARGUMENT); - }); - } - }); + error = await test.insertOne({ _id }).catch(error => error); - describe('enableUtf8Validation option set to true', () => { - // define client and option for tests to use - const option = { enableUtf8Validation: true }; - for (const passOptionTo of ['client', 'db', 'collection', 'operation']) { - it(`should enable validation with option passed to ${passOptionTo}`, async function () { - client = this.configuration.newClient(passOptionTo === 'client' ? option : undefined); - await client.connect(); - - const db = client.db('bson_utf8Validation_db', passOptionTo === 'db' ? option : undefined); - const collection = db.collection( - 'bson_utf8Validation_coll', - passOptionTo === 'collection' ? option : undefined - ); - - await collection.insertOne( - { name: 'John Doe' }, - passOptionTo === 'operation' ? option : {} - ); - - expect(bsonSpy).to.have.been.called; - const result = bsonSpy.lastCall.returnValue; - expect(result).to.deep.equal(EXPECTED_VALIDATION_ENABLED_ARGUMENT); - }); - } - }); + // Check that the server sent us broken BSON (bad UTF) + expect(() => { + BSON.deserialize(spy.returnValues[0], { validation: { utf8: true } }); + }).to.throw(BSON.BSONError, /Invalid UTF/i, 'did not generate error with invalid utf8'); + } - describe('enableUtf8Validation option not set', () => { - const option = {}; - for (const passOptionTo of ['client', 'db', 'collection', 'operation']) { - it(`should default to enabled with option passed to ${passOptionTo}`, async function () { - client = this.configuration.newClient(passOptionTo === 'client' ? option : undefined); - await client.connect(); - - const db = client.db('bson_utf8Validation_db', passOptionTo === 'db' ? option : undefined); - const collection = db.collection( - 'bson_utf8Validation_coll', - passOptionTo === 'collection' ? option : undefined - ); - - await collection.insertOne( - { name: 'John Doe' }, - passOptionTo === 'operation' ? option : {} - ); - - expect(bsonSpy).to.have.been.called; - const result = bsonSpy.lastCall.returnValue; - expect(result).to.deep.equal(EXPECTED_VALIDATION_ENABLED_ARGUMENT); - }); - } - }); - - context( - 'when the server is given a long multibyte utf sequence and there is a writeError', - () => { - let client: MongoClient; - beforeEach(async function () { - client = this.configuration.newClient(); + await generateWriteErrorWithInvalidUtf8(); }); afterEach(async function () { @@ -133,22 +55,7 @@ describe('class MongoDBResponse', () => { await client.close(); }); - it('does not throw a UTF-8 parsing error', async () => { - // Insert a large string of multibyte UTF-8 characters - const _id = '\u{1F92A}'.repeat(100); - - const test = client.db('parsing').collection<{ _id: string }>('parsing'); - await test.insertOne({ _id }); - - const spy = sinon.spy(OpMsgResponse.prototype, 'parse'); - - const error = await test.insertOne({ _id }).catch(error => error); - - // Check that the server sent us broken BSON (bad UTF) - expect(() => { - BSON.deserialize(spy.returnValues[0], { validation: { utf8: true } }); - }).to.throw(BSON.BSONError, /Invalid UTF/i); - + it('does not throw a UTF-8 parsing error', function () { // Assert the driver squashed it expect(error).to.be.instanceOf(MongoServerError); expect(error.message).to.match(/duplicate/i); @@ -176,7 +83,9 @@ describe('utf8 validation with cursors', function () { if (providedBuffer.includes(targetBytes)) { if (providedBuffer.split(targetBytes).length !== 2) { sinon.restore(); - const message = `too many target bytes sequences: received ${providedBuffer.split(targetBytes).length}\n. command: ${inspect(deserialize(args[0]), { depth: Infinity })}`; + const message = `too many target bytes sequences: received ${ + providedBuffer.split(targetBytes).length + }\n. command: ${inspect(deserialize(args[0]), { depth: Infinity })}`; throw new Error(message); } const buffer = Buffer.from(providedBuffer.replace(targetBytes, 'c301'.repeat(8)), 'hex'); diff --git a/test/unit/cmap/wire_protocol/responses.test.ts b/test/unit/cmap/wire_protocol/responses.test.ts index 7fccbfc7fc..9f2527d8a3 100644 --- a/test/unit/cmap/wire_protocol/responses.test.ts +++ b/test/unit/cmap/wire_protocol/responses.test.ts @@ -1,9 +1,5 @@ import { expect } from 'chai'; -import * as sinon from 'sinon'; -// to spy on the bson module, we must import it from the driver -// eslint-disable-next-line @typescript-eslint/no-restricted-imports -import * as mdb from '../../../../src/bson'; import { CursorResponse, Int32, @@ -17,74 +13,6 @@ describe('class MongoDBResponse', () => { it('is a subclass of OnDemandDocument', () => { expect(new MongoDBResponse(serialize({ ok: 1 }))).to.be.instanceOf(OnDemandDocument); }); - - context('utf8 validation', () => { - let deseriailzeSpy: sinon.SinonStub>; - beforeEach(function () { - const deserialize = mdb.deserialize; - deseriailzeSpy = sinon.stub>().callsFake(deserialize); - sinon.stub(mdb, 'deserialize').get(() => { - return deseriailzeSpy; - }); - }); - afterEach(function () { - sinon.restore(); - }); - - context('when enableUtf8Validation is not specified', () => { - const options = { enableUtf8Validation: undefined }; - it('calls BSON deserialize with writeErrors validation turned off', () => { - const res = new MongoDBResponse(serialize({})); - res.toObject(options); - - expect(deseriailzeSpy).to.have.been.called; - - const [ - { - args: [_buffer, { validation }] - } - ] = deseriailzeSpy.getCalls(); - - expect(validation).to.deep.equal({ utf8: { writeErrors: false } }); - }); - }); - - context('when enableUtf8Validation is true', () => { - const options = { enableUtf8Validation: true }; - it('calls BSON deserialize with writeErrors validation turned off', () => { - const res = new MongoDBResponse(serialize({})); - res.toObject(options); - - expect(deseriailzeSpy).to.have.been.called; - - const [ - { - args: [_buffer, { validation }] - } - ] = deseriailzeSpy.getCalls(); - - expect(validation).to.deep.equal({ utf8: { writeErrors: false } }); - }); - }); - - context('when enableUtf8Validation is false', () => { - const options = { enableUtf8Validation: false }; - it('calls BSON deserialize with all validation disabled', () => { - const res = new MongoDBResponse(serialize({})); - res.toObject(options); - - expect(deseriailzeSpy).to.have.been.called; - - const [ - { - args: [_buffer, { validation }] - } - ] = deseriailzeSpy.getCalls(); - - expect(validation).to.deep.equal({ utf8: false }); - }); - }); - }); }); describe('class CursorResponse', () => {