perf(NODE-6246): Use buffer pool for ObjectId to significantly improve memory usage

src/objectid.ts

@@ -4,6 +4,33 @@ import { type InspectFn, defaultInspect } from './parser/utils';
 import { ByteUtils } from './utils/byte_utils';
 import { NumberUtils } from './utils/number_utils';
 
+// Settings for ObjectId Buffer pool
+// Disable pool by default in order to ensure compatibility
+// Specify larger poolSize to enable pool
+let currentPool: Uint8Array | null = null;
+let poolSize = 1; // Disable pool by default.
+let currentPoolOffset = 0;
+
+/**
+ * Retrieves a ObjectId pool and offset. This function may create a new ObjectId buffer pool and reset the pool offset
+ * @internal
+ */
+function getPool(): [Uint8Array, number] {
+ if (!currentPool || currentPoolOffset + 12 > currentPool.length) {
+ currentPool = ByteUtils.allocateUnsafe(poolSize * 12);
+ currentPoolOffset = 0;
+ }
+ return [currentPool, currentPoolOffset];
+}
+
+/**
+ * Increments the pool offset by 12 bytes
+ * @internal
+ */
+function incrementPool(): void {
+ currentPoolOffset += 12;
+}
+
 // Regular expression that checks for hex value
 const checkForHexRegExp = new RegExp('^[0-9a-fA-F]{24}$');
 
@@ -37,8 +64,22 @@ export class ObjectId extends BSONValue {
 
  static cacheHexString: boolean;
 
- /** ObjectId Bytes @internal */
- private buffer!: Uint8Array;
+ /**
+ * The size of the current ObjectId buffer pool.
+ */
+ static get poolSize(): number {
+ return poolSize;
+ }
+
+ static set poolSize(size: number) {
+ poolSize = Math.max(Math.abs(Number(size)) >>> 0, 1);
+ }
+
+ /** ObjectId buffer pool pointer @internal */
+ private pool: Uint8Array;
+ /** Buffer pool offset @internal */
+ private offset?: number;
+
  /** ObjectId hexString cache @internal */
  private __id?: string;
 
@@ -73,6 +114,13 @@ export class ObjectId extends BSONValue {
  * @param inputId - A 12 byte binary Buffer.
  */
  constructor(inputId: Uint8Array);
+ /**
+ * Create ObjectId from a large binary Buffer. Only 12 bytes starting from the offset are used.
+ * @internal
+ * @param inputId - A 12 byte binary Buffer.
+ * @param inputIndex - The offset to start reading the inputId buffer.
+ */
+ constructor(inputId: Uint8Array, inputIndex?: number);
  /** To generate a new ObjectId, use ObjectId() with no argument. */
  constructor();
  /**
@@ -86,7 +134,10 @@ export class ObjectId extends BSONValue {
  *
  * @param inputId - An input value to create a new ObjectId from.
  */
- constructor(inputId?: string | number | ObjectId | ObjectIdLike | Uint8Array) {
+ constructor(
+ inputId?: string | number | ObjectId | ObjectIdLike | Uint8Array,
+ inputIndex?: number
+ ) {
  super();
  // workingId is set based on type of input and whether valid id exists for the input
  let workingId;
@@ -103,41 +154,77 @@ export class ObjectId extends BSONValue {
  workingId = inputId;
  }
 
- // The following cases use workingId to construct an ObjectId
- if (workingId == null || typeof workingId === 'number') {
- // The most common use case (blank id, new objectId instance)
- // Generate a new id
- this.buffer = ObjectId.generate(typeof workingId === 'number' ? workingId : undefined);
- } else if (ArrayBuffer.isView(workingId) && workingId.byteLength === 12) {
- // If intstanceof matches we can escape calling ensure buffer in Node.js environments
- this.buffer = ByteUtils.toLocalBufferType(workingId);
- } else if (typeof workingId === 'string') {
- if (workingId.length === 24 && checkForHexRegExp.test(workingId)) {
- this.buffer = ByteUtils.fromHex(workingId);
+ let pool: Uint8Array;
+ let offset: number;
+
+ // Special case when poolSize === 1 and a 12 byte buffer is passed in - just persist buffer
+ if (poolSize === 1 && ArrayBuffer.isView(workingId) && workingId.length === 12) {
+ pool = ByteUtils.toLocalBufferType(workingId);
+ offset = 0;
+ } else {
+ [pool, offset] = getPool();
+
+ // The following cases use workingId to construct an ObjectId
+ if (workingId == null || typeof workingId === 'number') {
+ // The most common use case (blank id, new objectId instance)
+ // Generate a new id
+ ObjectId.generate(typeof workingId === 'number' ? workingId : undefined, pool, offset);
+ } else if (ArrayBuffer.isView(workingId)) {
+ if (workingId.length === 12) {
+ inputIndex = 0;
+ } else if (
+ typeof inputIndex !== 'number' ||
+ inputIndex < 0 ||
+ workingId.length < inputIndex + 12 ||
+ isNaN(inputIndex)
+ ) {
+ throw new BSONError('Buffer length must be 12 or a valid offset must be specified');
+ }
+ for (let i = 0; i < 12; i++) pool[offset + i] = workingId[inputIndex + i];
+ } else if (typeof workingId === 'string') {
+ if (workingId.length === 24 && checkForHexRegExp.test(workingId)) {
+ pool.set(ByteUtils.fromHex(workingId), offset);
+ } else {
+ throw new BSONError(
+ 'input must be a 24 character hex string, 12 byte Uint8Array, or an integer'
+ );
+ }
  } else {
- throw new BSONError(
- 'input must be a 24 character hex string, 12 byte Uint8Array, or an integer'
- );
+ throw new BSONError('Argument passed in does not match the accepted types');
  }
- } else {
- throw new BSONError('Argument passed in does not match the accepted types');
  }
  // If we are caching the hex string
  if (ObjectId.cacheHexString) {
- this.__id = ByteUtils.toHex(this.id);
+ this.__id = ByteUtils.toHex(pool, offset, offset + 12);
+ }
+ // Increment pool offset once we have completed initialization
+ this.pool = pool;
+ // Only set offset if pool is used
+ if (poolSize > 1) {
+ this.offset = offset;
  }
+ incrementPool();
+ }
+
+ /** ObjectId bytes @internal */
+ get buffer(): Uint8Array {
+ return this.id;
  }
 
  /**
  * The ObjectId bytes
  * @readonly
  */
  get id(): Uint8Array {
- return this.buffer;
+ if (this.offset === undefined) return this.pool;
+ return this.pool.subarray(this.offset, this.offset + 12);
  }
 
  set id(value: Uint8Array) {
- this.buffer = value;
+ if (value.byteLength !== 12) {
+ throw new BSONError('input must be a 12 byte Uint8Array');
+ }
+ this.pool.set(value, this.offset);
  if (ObjectId.cacheHexString) {
  this.__id = ByteUtils.toHex(value);
  }
@@ -148,8 +235,9 @@ export class ObjectId extends BSONValue {
  if (ObjectId.cacheHexString && this.__id) {
  return this.__id;
  }
+ const start = this.offset ?? 0;
 
- const hexString = ByteUtils.toHex(this.id);
+ const hexString = ByteUtils.toHex(this.pool, start, start + 12);
 
  if (ObjectId.cacheHexString && !this.__id) {
  this.__id = hexString;
@@ -171,33 +259,52 @@ export class ObjectId extends BSONValue {
  *
  * @param time - pass in a second based timestamp.
  */
- static generate(time?: number): Uint8Array {
+ static generate(time?: number): Uint8Array;
+ /**
+ * Generate a 12 byte id buffer used in ObjectId's and write to the provided buffer at offset.
+ * @internal
+ *
+ * @param time - pass in a second based timestamp.
+ * @param buffer - Optionally pass in a buffer instance.
+ * @param offset - Optionally pass in a buffer offset.
+ */
+ static generate(time?: number, buffer?: Uint8Array, offset?: number): Uint8Array;
+ /**
+ * Generate a 12 byte id buffer used in ObjectId's
+ *
+ * @param time - pass in a second based timestamp.
+ * @param buffer - Optionally pass in a buffer instance.
+ * @param offset - Optionally pass in a buffer offset.
+ */
+ static generate(time?: number, buffer?: Uint8Array, offset: number = 0): Uint8Array {
  if ('number' !== typeof time) {
  time = Math.floor(Date.now() / 1000);
  }
 
  const inc = ObjectId.getInc();
- const buffer = ByteUtils.allocateUnsafe(12);
+ if (!buffer) {
+ buffer = ByteUtils.allocateUnsafe(12);
+ }
 
  // 4-byte timestamp
- NumberUtils.setInt32BE(buffer, 0, time);
+ NumberUtils.setInt32BE(buffer, offset, time);
 
  // set PROCESS_UNIQUE if yet not initialized
  if (PROCESS_UNIQUE === null) {
  PROCESS_UNIQUE = ByteUtils.randomBytes(5);
  }
 
  // 5-byte process unique
- buffer[4] = PROCESS_UNIQUE[0];
- buffer[5] = PROCESS_UNIQUE[1];
- buffer[6] = PROCESS_UNIQUE[2];
- buffer[7] = PROCESS_UNIQUE[3];
- buffer[8] = PROCESS_UNIQUE[4];
+ buffer[offset + 4] = PROCESS_UNIQUE[0];
+ buffer[offset + 5] = PROCESS_UNIQUE[1];
+ buffer[offset + 6] = PROCESS_UNIQUE[2];
+ buffer[offset + 7] = PROCESS_UNIQUE[3];
+ buffer[offset + 8] = PROCESS_UNIQUE[4];
 
  // 3-byte counter
- buffer[11] = inc & 0xff;
- buffer[10] = (inc >> 8) & 0xff;
- buffer[9] = (inc >> 16) & 0xff;
+ buffer[offset + 11] = inc & 0xff;
+ buffer[offset + 10] = (inc >> 8) & 0xff;
+ buffer[offset + 9] = (inc >> 16) & 0xff;
 
  return buffer;
  }
@@ -239,6 +346,17 @@ export class ObjectId extends BSONValue {
  }
 
  if (ObjectId.is(otherId)) {
+ if (otherId.pool) {
+ for (let i = 11; i >= 0; i--) {
+ const offset = this.offset ?? 0;
+ const otherOffset = otherId.offset ?? 0;
+ if (this.pool[offset + i] !== otherId.pool[otherOffset + i]) {
+ return false;
+ }
+ }
+ return true;
+ }
+ // If otherId does not have pool and offset, fallback to buffer comparison for compatibility
  return (
  this.buffer[11] === otherId.buffer[11] && ByteUtils.equals(this.buffer, otherId.buffer)
  );
@@ -260,7 +378,7 @@ export class ObjectId extends BSONValue {
  /** Returns the generation date (accurate up to the second) that this ID was generated. */
  getTimestamp(): Date {
  const timestamp = new Date();
- const time = NumberUtils.getUint32BE(this.buffer, 0);
+ const time = NumberUtils.getUint32BE(this.pool, this.offset ?? 0);
  timestamp.setTime(Math.floor(time) * 1000);
  return timestamp;
  }
@@ -272,18 +390,20 @@ export class ObjectId extends BSONValue {
 
  /** @internal */
  serializeInto(uint8array: Uint8Array, index: number): 12 {
- uint8array[index] = this.buffer[0];
- uint8array[index + 1] = this.buffer[1];
- uint8array[index + 2] = this.buffer[2];
- uint8array[index + 3] = this.buffer[3];
- uint8array[index + 4] = this.buffer[4];
- uint8array[index + 5] = this.buffer[5];
- uint8array[index + 6] = this.buffer[6];
- uint8array[index + 7] = this.buffer[7];
- uint8array[index + 8] = this.buffer[8];
- uint8array[index + 9] = this.buffer[9];
- uint8array[index + 10] = this.buffer[10];
- uint8array[index + 11] = this.buffer[11];
+ const pool = this.pool;
+ const offset = this.offset ?? 0;
+ uint8array[index] = pool[offset];
+ uint8array[index + 1] = pool[offset + 1];
+ uint8array[index + 2] = pool[offset + 2];
+ uint8array[index + 3] = pool[offset + 3];
+ uint8array[index + 4] = pool[offset + 4];
+ uint8array[index + 5] = pool[offset + 5];
+ uint8array[index + 6] = pool[offset + 6];
+ uint8array[index + 7] = pool[offset + 7];
+ uint8array[index + 8] = pool[offset + 8];
+ uint8array[index + 9] = pool[offset + 9];
+ uint8array[index + 10] = pool[offset + 10];
+ uint8array[index + 11] = pool[offset + 11];
  return 12;
  }
 
@@ -293,7 +413,7 @@ export class ObjectId extends BSONValue {
  * @param time - an integer number representing a number of seconds.
  */
  static createFromTime(time: number): ObjectId {
- const buffer = ByteUtils.allocate(12);
+ const buffer = ByteUtils.allocateUnsafe(12);
  for (let i = 11; i >= 4; i--) buffer[i] = 0;
  // Encode time into first 4 bytes
  NumberUtils.setInt32BE(buffer, 0, time);

src/parser/deserializer.ts

@@ -263,9 +263,7 @@ function deserializeObject(
  value = ByteUtils.toUTF8(buffer, index, index + stringSize - 1, shouldValidateKey);
  index = index + stringSize;
  } else if (elementType === constants.BSON_DATA_OID) {
- const oid = ByteUtils.allocateUnsafe(12);
- for (let i = 0; i < 12; i++) oid[i] = buffer[index + i];
- value = new ObjectId(oid);
+ value = new ObjectId(buffer, index);
  index = index + 12;
  } else if (elementType === constants.BSON_DATA_INT && promoteValues === false) {
  value = new Int32(NumberUtils.getInt32LE(buffer, index));

src/utils/byte_utils.ts

@@ -30,7 +30,7 @@ export type ByteUtils = {
  /** Create a Uint8Array from a hex string */
  fromHex: (hex: string) => Uint8Array;
  /** Create a lowercase hex string from bytes */
- toHex: (buffer: Uint8Array) => string;
+ toHex: (buffer: Uint8Array, start?: number, end?: number) => string;
  /** Create a string from utf8 code units, fatal=true will throw an error if UTF-8 bytes are invalid, fatal=false will insert replacement characters */
  toUTF8: (buffer: Uint8Array, start: number, end: number, fatal: boolean) => string;
  /** Get the utf8 code unit count from a string if it were to be transformed to utf8 */

src/utils/node_byte_utils.ts

@@ -124,8 +124,8 @@ export const nodeJsByteUtils = {
  return Buffer.from(hex, 'hex');
  },
 
- toHex(buffer: Uint8Array): string {
- return nodeJsByteUtils.toLocalBufferType(buffer).toString('hex');
+ toHex(buffer: Uint8Array, start?: number, end?: number): string {
+ return nodeJsByteUtils.toLocalBufferType(buffer).toString('hex', start, end);
  },
 
  toUTF8(buffer: Uint8Array, start: number, end: number, fatal: boolean): string {

src/utils/web_byte_utils.ts

@@ -170,8 +170,10 @@ export const webByteUtils = {
  return Uint8Array.from(buffer);
  },
 
- toHex(uint8array: Uint8Array): string {
- return Array.from(uint8array, byte => byte.toString(16).padStart(2, '0')).join('');
+ toHex(uint8array: Uint8Array, start?: number, end?: number): string {
+ return Array.from(uint8array.subarray(start, end), byte =>
+ byte.toString(16).padStart(2, '0')
+ ).join('');
  },
 
  toUTF8(uint8array: Uint8Array, start: number, end: number, fatal: boolean): string {

test/node/bson_test.js

@@ -707,7 +707,7 @@ describe('BSON', function () {
  expect(serialized_data).to.deep.equal(serialized_data2);
 
  var doc2 = b.deserialize(serialized_data);
- expect(doc).to.deep.equal(doc2);
+ expect(b.serialize(doc)).to.deep.equal(b.serialize(doc2));
  expect(doc2.dbref.oid.toHexString()).to.deep.equal(oid.toHexString());
  done();
  });
@@ -1001,7 +1001,7 @@ describe('BSON', function () {
 
  var deserialized_data = BSON.deserialize(serialized_data);
  expect(doc.b).to.deep.equal(deserialized_data.b);
- expect(doc).to.deep.equal(deserialized_data);
+ expect(BSON.serialize(doc)).to.deep.equal(BSON.serialize(deserialized_data));
  done();
  });
 
@@ -1213,7 +1213,7 @@ describe('BSON', function () {
 
  var doc2 = BSON.deserialize(serialized_data);
 
- expect(doc).to.deep.equal(doc2);
+ expect(BSON.serialize(doc)).to.deep.equal(BSON.serialize(doc2));
  done();
  });