From ab6cf01b6765cff97041287915502e5361ed556e Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Tue, 15 Oct 2024 10:18:11 +0700 Subject: [PATCH] feat: improve type.hashTreeRoot() using batch --- packages/as-sha256/src/index.ts | 1 + packages/ssz/src/type/abstract.ts | 5 + packages/ssz/src/type/arrayComposite.ts | 26 +++-- packages/ssz/src/type/basic.ts | 13 ++- packages/ssz/src/type/bitArray.ts | 13 ++- packages/ssz/src/type/bitList.ts | 30 +++++- packages/ssz/src/type/byteArray.ts | 31 +++++- packages/ssz/src/type/byteList.ts | 31 +++++- packages/ssz/src/type/composite.ts | 35 ++++++- packages/ssz/src/type/container.ts | 13 +-- packages/ssz/src/type/containerNodeStruct.ts | 1 - packages/ssz/src/type/listBasic.ts | 61 +++++++++--- packages/ssz/src/type/listComposite.ts | 54 ++++++++--- packages/ssz/src/type/optional.ts | 34 +++++-- packages/ssz/src/type/profile.ts | 37 +++++--- packages/ssz/src/type/stableContainer.ts | 79 +++++++++------- packages/ssz/src/type/uint.ts | 6 ++ packages/ssz/src/type/union.ts | 29 ++++-- packages/ssz/src/type/vectorBasic.ts | 16 +++- packages/ssz/src/type/vectorComposite.ts | 10 +- packages/ssz/src/util/merkleize.ts | 41 ++++---- .../ssz/test/perf/eth2/beaconBlock.test.ts | 94 +++++++++++++++++++ packages/ssz/test/perf/merkleize.test.ts | 20 +++- packages/ssz/test/spec/runValidTest.ts | 9 +- packages/ssz/test/unit/merkleize.test.ts | 35 ++++++- 25 files changed, 565 insertions(+), 159 deletions(-) create mode 100644 packages/ssz/test/perf/eth2/beaconBlock.test.ts diff --git a/packages/as-sha256/src/index.ts b/packages/as-sha256/src/index.ts index 11f44cc3..46d60789 100644 --- a/packages/as-sha256/src/index.ts +++ b/packages/as-sha256/src/index.ts @@ -3,6 +3,7 @@ import {newInstance} from "./wasm"; import {HashObject, byteArrayIntoHashObject, byteArrayToHashObject, hashObjectToByteArray} from "./hashObject"; import SHA256 from "./sha256"; export {HashObject, byteArrayToHashObject, hashObjectToByteArray, byteArrayIntoHashObject, SHA256}; +export {allocUnsafe}; const ctx = newInstance(); const wasmInputValue = ctx.input.value; diff --git a/packages/ssz/src/type/abstract.ts b/packages/ssz/src/type/abstract.ts index b96b7355..792ca077 100644 --- a/packages/ssz/src/type/abstract.ts +++ b/packages/ssz/src/type/abstract.ts @@ -145,6 +145,11 @@ export abstract class Type { */ abstract hashTreeRoot(value: V): Uint8Array; + /** + * Same to hashTreeRoot() but here we write result to output. + */ + abstract hashTreeRootInto(value: V, output: Uint8Array, offset: number): void; + // JSON support /** Parse JSON representation of a type to value */ diff --git a/packages/ssz/src/type/arrayComposite.ts b/packages/ssz/src/type/arrayComposite.ts index d3b0a8fb..986b0e0a 100644 --- a/packages/ssz/src/type/arrayComposite.ts +++ b/packages/ssz/src/type/arrayComposite.ts @@ -211,21 +211,29 @@ export function tree_deserializeFromBytesArrayComposite>( +export function value_getChunkBytesArrayComposite>( elementType: ElementType, length: number, - value: ValueOf[] -): Uint8Array[] { - const roots = new Array(length); + value: ValueOf[], + chunkBytesBuffer: Uint8Array +): Uint8Array { + const isOddChunk = length % 2 === 1; + const chunkBytesLen = isOddChunk ? length * 32 + 32 : length * 32; + if (chunkBytesLen > chunkBytesBuffer.length) { + throw new Error(`chunkBytesBuffer is too small: ${chunkBytesBuffer.length} < ${chunkBytesLen}`); + } + const chunkBytes = chunkBytesBuffer.subarray(0, chunkBytesLen); for (let i = 0; i < length; i++) { - roots[i] = elementType.hashTreeRoot(value[i]); + elementType.hashTreeRootInto(value[i], chunkBytes, i * 32); + } + + if (isOddChunk) { + // similar to append zeroHash(0) + chunkBytes.subarray(length * 32, chunkBytesLen).fill(0); } - return roots; + return chunkBytes; } function readOffsetsArrayComposite( diff --git a/packages/ssz/src/type/basic.ts b/packages/ssz/src/type/basic.ts index 0260ea49..920c6d97 100644 --- a/packages/ssz/src/type/basic.ts +++ b/packages/ssz/src/type/basic.ts @@ -30,11 +30,18 @@ export abstract class BasicType extends Type { } hashTreeRoot(value: V): Uint8Array { - // TODO: Optimize - const uint8Array = new Uint8Array(32); + // cannot use allocUnsafe() here because hashTreeRootInto() may not fill the whole 32 bytes + const root = new Uint8Array(32); + this.hashTreeRootInto(value, root, 0); + return root; + } + + hashTreeRootInto(value: V, output: Uint8Array, offset: number): void { + const uint8Array = output.subarray(offset, offset + 32); + // output could have preallocated data, some types may not fill the whole 32 bytes + uint8Array.fill(0); const dataView = new DataView(uint8Array.buffer, uint8Array.byteOffset, uint8Array.byteLength); this.value_serializeToBytes({uint8Array, dataView}, 0, value); - return uint8Array; } clone(value: V): V { diff --git a/packages/ssz/src/type/bitArray.ts b/packages/ssz/src/type/bitArray.ts index 5071550c..d485de27 100644 --- a/packages/ssz/src/type/bitArray.ts +++ b/packages/ssz/src/type/bitArray.ts @@ -1,10 +1,10 @@ import {concatGindices, Gindex, Node, toGindex, Tree, HashComputationLevel} from "@chainsafe/persistent-merkle-tree"; import {fromHexString, toHexString, byteArrayEquals} from "../util/byteArray"; -import {splitIntoRootChunks} from "../util/merkleize"; import {CompositeType, LENGTH_GINDEX} from "./composite"; import {BitArray} from "../value/bitArray"; import {BitArrayTreeView} from "../view/bitArray"; import {BitArrayTreeViewDU} from "../viewDU/bitArray"; +import {getChunkBytes} from "./byteArray"; /* eslint-disable @typescript-eslint/member-ordering */ @@ -40,8 +40,15 @@ export abstract class BitArrayType extends CompositeType this.chunkBytesBuffer.length) { + const chunkCount = Math.ceil(value.bitLen / 8 / 32); + const chunkBytes = chunkCount * 32; + // pad 1 chunk if maxChunkCount is not even + this.chunkBytesBuffer = chunkCount % 2 === 1 ? new Uint8Array(chunkBytes + 32) : new Uint8Array(chunkBytes); + } + return getChunkBytes(value.uint8Array, this.chunkBytesBuffer); } // Proofs diff --git a/packages/ssz/src/type/bitList.ts b/packages/ssz/src/type/bitList.ts index 0d8268b2..ba1c419a 100644 --- a/packages/ssz/src/type/bitList.ts +++ b/packages/ssz/src/type/bitList.ts @@ -1,5 +1,12 @@ -import {getNodesAtDepth, Node, packedNodeRootsToBytes, packedRootsBytesToNode} from "@chainsafe/persistent-merkle-tree"; -import {mixInLength, maxChunksToDepth} from "../util/merkleize"; +import {allocUnsafe} from "@chainsafe/as-sha256"; +import { + getNodesAtDepth, + merkleizeInto, + Node, + packedNodeRootsToBytes, + packedRootsBytesToNode, +} from "@chainsafe/persistent-merkle-tree"; +import {maxChunksToDepth} from "../util/merkleize"; import {Require} from "../util/types"; import {namedClass} from "../util/named"; import {ByteViews} from "./composite"; @@ -29,6 +36,12 @@ export class BitListType extends BitArrayType { readonly maxSize: number; readonly maxChunkCount: number; readonly isList = true; + readonly mixInLengthChunkBytes = new Uint8Array(64); + readonly mixInLengthBuffer = Buffer.from( + this.mixInLengthChunkBytes.buffer, + this.mixInLengthChunkBytes.byteOffset, + this.mixInLengthChunkBytes.byteLength + ); constructor(readonly limitBits: number, opts?: BitListOptions) { super(); @@ -101,7 +114,18 @@ export class BitListType extends BitArrayType { // Merkleization: inherited from BitArrayType hashTreeRoot(value: BitArray): Uint8Array { - return mixInLength(super.hashTreeRoot(value), value.bitLen); + const root = allocUnsafe(32); + this.hashTreeRootInto(value, root, 0); + return root; + } + + hashTreeRootInto(value: BitArray, output: Uint8Array, offset: number): void { + super.hashTreeRootInto(value, this.mixInLengthChunkBytes, 0); + // mixInLength + this.mixInLengthBuffer.writeUIntLE(value.bitLen, 32, 6); + // one for hashTreeRoot(value), one for length + const chunkCount = 2; + merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset); } // Proofs: inherited from BitArrayType diff --git a/packages/ssz/src/type/byteArray.ts b/packages/ssz/src/type/byteArray.ts index 202f1b24..78e6ae30 100644 --- a/packages/ssz/src/type/byteArray.ts +++ b/packages/ssz/src/type/byteArray.ts @@ -8,7 +8,6 @@ import { getHashComputations, } from "@chainsafe/persistent-merkle-tree"; import {fromHexString, toHexString, byteArrayEquals} from "../util/byteArray"; -import {splitIntoRootChunks} from "../util/merkleize"; import {ByteViews} from "./abstract"; import {CompositeType, LENGTH_GINDEX} from "./composite"; @@ -82,10 +81,23 @@ export abstract class ByteArrayType extends CompositeType this.chunkBytesBuffer.length) { + const chunkCount = Math.ceil(value.length / 32); + const chunkBytes = chunkCount * 32; + // pad 1 chunk if maxChunkCount is not even + this.chunkBytesBuffer = chunkCount % 2 === 1 ? new Uint8Array(chunkBytes + 32) : new Uint8Array(chunkBytes); + } + return getChunkBytes(value, this.chunkBytesBuffer); } // Proofs @@ -149,3 +161,16 @@ export abstract class ByteArrayType extends CompositeType merkleBytesBuffer.length) { + throw new Error(`data length ${data.length} exceeds merkleBytesBuffer length ${merkleBytesBuffer.length}`); + } + + merkleBytesBuffer.set(data); + const valueLen = data.length; + const chunkByteLen = Math.ceil(valueLen / 64) * 64; + // all padding bytes must be zero, this is similar to set zeroHash(0) + merkleBytesBuffer.subarray(valueLen, chunkByteLen).fill(0); + return merkleBytesBuffer.subarray(0, chunkByteLen); +} diff --git a/packages/ssz/src/type/byteList.ts b/packages/ssz/src/type/byteList.ts index 6f12fff7..53c46d39 100644 --- a/packages/ssz/src/type/byteList.ts +++ b/packages/ssz/src/type/byteList.ts @@ -1,11 +1,17 @@ -import {getNodesAtDepth, Node, packedNodeRootsToBytes, packedRootsBytesToNode} from "@chainsafe/persistent-merkle-tree"; -import {mixInLength, maxChunksToDepth} from "../util/merkleize"; +import {allocUnsafe} from "@chainsafe/as-sha256"; +import { + getNodesAtDepth, + Node, + packedNodeRootsToBytes, + packedRootsBytesToNode, + merkleizeInto, +} from "@chainsafe/persistent-merkle-tree"; +import {maxChunksToDepth} from "../util/merkleize"; import {Require} from "../util/types"; import {namedClass} from "../util/named"; import {addLengthNode, getChunksNodeFromRootNode, getLengthFromRootNode} from "./arrayBasic"; import {ByteViews} from "./composite"; import {ByteArrayType, ByteArray} from "./byteArray"; - /* eslint-disable @typescript-eslint/member-ordering */ export interface ByteListOptions { @@ -34,6 +40,12 @@ export class ByteListType extends ByteArrayType { readonly maxSize: number; readonly maxChunkCount: number; readonly isList = true; + readonly mixInLengthChunkBytes = new Uint8Array(64); + readonly mixInLengthBuffer = Buffer.from( + this.mixInLengthChunkBytes.buffer, + this.mixInLengthChunkBytes.byteOffset, + this.mixInLengthChunkBytes.byteLength + ); constructor(readonly limitBytes: number, opts?: ByteListOptions) { super(); @@ -89,7 +101,18 @@ export class ByteListType extends ByteArrayType { // Merkleization: inherited from ByteArrayType hashTreeRoot(value: ByteArray): Uint8Array { - return mixInLength(super.hashTreeRoot(value), value.length); + const root = allocUnsafe(32); + this.hashTreeRootInto(value, root, 0); + return root; + } + + hashTreeRootInto(value: Uint8Array, output: Uint8Array, offset: number): void { + super.hashTreeRootInto(value, this.mixInLengthChunkBytes, 0); + // mixInLength + this.mixInLengthBuffer.writeUIntLE(value.length, 32, 6); + // one for hashTreeRoot(value), one for length + const chunkCount = 2; + merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset); } // Proofs: inherited from BitArrayType diff --git a/packages/ssz/src/type/composite.ts b/packages/ssz/src/type/composite.ts index c403c385..9f33ca5e 100644 --- a/packages/ssz/src/type/composite.ts +++ b/packages/ssz/src/type/composite.ts @@ -1,3 +1,4 @@ +import {allocUnsafe} from "@chainsafe/as-sha256"; import { concatGindices, createProof, @@ -7,10 +8,11 @@ import { Proof, ProofType, Tree, + merkleizeInto, HashComputationLevel, } from "@chainsafe/persistent-merkle-tree"; import {byteArrayEquals} from "../util/byteArray"; -import {merkleize, symbolCachedPermanentRoot, ValueWithCachedPermanentRoot} from "../util/merkleize"; +import {cacheRoot, symbolCachedPermanentRoot, ValueWithCachedPermanentRoot} from "../util/merkleize"; import {treePostProcessFromProofNode} from "../util/proof/treePostProcessFromProofNode"; import {Type, ByteViews, JsonPath, JsonPathProp} from "./abstract"; export {ByteViews}; @@ -59,6 +61,7 @@ export abstract class CompositeType extends Type { * Required for ContainerNodeStruct to ensure no dangerous types are constructed. */ abstract readonly isViewMutable: boolean; + protected chunkBytesBuffer = new Uint8Array(0); constructor( /** @@ -216,13 +219,30 @@ export abstract class CompositeType extends Type { } } - const root = merkleize(this.getRoots(value), this.maxChunkCount); + const root = allocUnsafe(32); + const safeCache = true; + this.hashTreeRootInto(value, root, 0, safeCache); + // hashTreeRootInto will cache the root if cachePermanentRootStruct is true + + return root; + } + + hashTreeRootInto(value: V, output: Uint8Array, offset: number, safeCache = false): void { + // Return cached mutable root if any if (this.cachePermanentRootStruct) { - (value as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot] = root; + const cachedRoot = (value as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot]; + if (cachedRoot) { + output.set(cachedRoot, offset); + return; + } } - return root; + const merkleBytes = this.getChunkBytes(value); + merkleizeInto(merkleBytes, this.maxChunkCount, output, offset); + if (this.cachePermanentRootStruct) { + cacheRoot(value as ValueWithCachedPermanentRoot, output, offset, safeCache); + } } // For debugging and testing this feature @@ -236,7 +256,12 @@ export abstract class CompositeType extends Type { // and feed those numbers directly to the hasher input with a DataView // - The return of the hasher should be customizable too, to reduce conversions from Uint8Array // to hashObject and back. - protected abstract getRoots(value: V): Uint8Array[]; + + /** + * Get merkle bytes of each value, the returned Uint8Array should be multiple of 64 bytes. + * If chunk count is not even, need to append zeroHash(0) + */ + protected abstract getChunkBytes(value: V): Uint8Array; // Proofs API diff --git a/packages/ssz/src/type/container.ts b/packages/ssz/src/type/container.ts index daa1911d..1ed46a89 100644 --- a/packages/ssz/src/type/container.ts +++ b/packages/ssz/src/type/container.ts @@ -130,6 +130,9 @@ export class ContainerType>> extends // Refactor this constructor to allow customization without pollutin the options this.TreeView = opts?.getContainerTreeViewClass?.(this) ?? getContainerTreeViewClass(this); this.TreeViewDU = opts?.getContainerTreeViewDUClass?.(this) ?? getContainerTreeViewDUClass(this); + const fieldBytes = this.fieldsEntries.length * 32; + const chunkBytes = Math.ceil(fieldBytes / 64) * 64; + this.chunkBytesBuffer = new Uint8Array(chunkBytes); } static named>>( @@ -272,15 +275,13 @@ export class ContainerType>> extends // Merkleization - protected getRoots(struct: ValueOfFields): Uint8Array[] { - const roots = new Array(this.fieldsEntries.length); - + protected getChunkBytes(struct: ValueOfFields): Uint8Array { for (let i = 0; i < this.fieldsEntries.length; i++) { const {fieldName, fieldType} = this.fieldsEntries[i]; - roots[i] = fieldType.hashTreeRoot(struct[fieldName]); + fieldType.hashTreeRootInto(struct[fieldName], this.chunkBytesBuffer, i * 32); } - - return roots; + // remaining bytes are zeroed as we never write them + return this.chunkBytesBuffer; } // Proofs diff --git a/packages/ssz/src/type/containerNodeStruct.ts b/packages/ssz/src/type/containerNodeStruct.ts index 8cefa381..76147f12 100644 --- a/packages/ssz/src/type/containerNodeStruct.ts +++ b/packages/ssz/src/type/containerNodeStruct.ts @@ -106,7 +106,6 @@ export class ContainerNodeStructType return new BranchNodeStruct(this.valueToTree.bind(this), value); } - // TODO: Optimize conversion private valueToTree(value: ValueOfFields): Node { const uint8Array = new Uint8Array(this.value_serializedSize(value)); const dataView = new DataView(uint8Array.buffer, uint8Array.byteOffset, uint8Array.byteLength); diff --git a/packages/ssz/src/type/listBasic.ts b/packages/ssz/src/type/listBasic.ts index c9e397e6..4dd63f08 100644 --- a/packages/ssz/src/type/listBasic.ts +++ b/packages/ssz/src/type/listBasic.ts @@ -1,4 +1,4 @@ -import {LeafNode, Node, Tree, HashComputationLevel} from "@chainsafe/persistent-merkle-tree"; +import {HashComputationLevel, LeafNode, Node, Tree, merkleizeInto} from "@chainsafe/persistent-merkle-tree"; import {ValueOf} from "./abstract"; import {BasicType} from "./basic"; import {ByteViews} from "./composite"; @@ -10,19 +10,14 @@ import { addLengthNode, setChunksNode, } from "./arrayBasic"; -import { - mixInLength, - maxChunksToDepth, - splitIntoRootChunks, - symbolCachedPermanentRoot, - ValueWithCachedPermanentRoot, -} from "../util/merkleize"; +import {cacheRoot, maxChunksToDepth, symbolCachedPermanentRoot, ValueWithCachedPermanentRoot} from "../util/merkleize"; import {Require} from "../util/types"; import {namedClass} from "../util/named"; import {ArrayBasicType} from "../view/arrayBasic"; import {ListBasicTreeView} from "../view/listBasic"; import {ListBasicTreeViewDU} from "../viewDU/listBasic"; import {ArrayType} from "./array"; +import {allocUnsafe} from "@chainsafe/as-sha256"; /* eslint-disable @typescript-eslint/member-ordering */ @@ -52,6 +47,12 @@ export class ListBasicType> readonly maxSize: number; readonly isList = true; readonly isViewMutable = true; + readonly mixInLengthChunkBytes = new Uint8Array(64); + readonly mixInLengthBuffer = Buffer.from( + this.mixInLengthChunkBytes.buffer, + this.mixInLengthChunkBytes.byteOffset, + this.mixInLengthChunkBytes.byteLength + ); protected readonly defaultLen = 0; constructor(readonly elementType: ElementType, readonly limit: number, opts?: ListBasicOpts) { @@ -174,20 +175,52 @@ export class ListBasicType> } } - const root = mixInLength(super.hashTreeRoot(value), value.length); + const root = allocUnsafe(32); + const safeCache = true; + this.hashTreeRootInto(value, root, 0, safeCache); + + // hashTreeRootInto will cache the root if cachePermanentRootStruct is true + return root; + } + + hashTreeRootInto(value: ValueOf[], output: Uint8Array, offset: number, safeCache = false): void { if (this.cachePermanentRootStruct) { - (value as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot] = root; + const cachedRoot = (value as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot]; + if (cachedRoot) { + output.set(cachedRoot, offset); + return; + } } - return root; + super.hashTreeRootInto(value, this.mixInLengthChunkBytes, 0); + // mixInLength + this.mixInLengthBuffer.writeUIntLE(value.length, 32, 6); + // one for hashTreeRoot(value), one for length + const chunkCount = 2; + merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset); + + if (this.cachePermanentRootStruct) { + cacheRoot(value as ValueWithCachedPermanentRoot, output, offset, safeCache); + } } - protected getRoots(value: ValueOf[]): Uint8Array[] { - const uint8Array = new Uint8Array(this.value_serializedSize(value)); + protected getChunkBytes(value: ValueOf[]): Uint8Array { + const byteLen = this.value_serializedSize(value); + const chunkByteLen = Math.ceil(byteLen / 64) * 64; + // reallocate this.verkleBytes if needed + if (byteLen > this.chunkBytesBuffer.length) { + // pad 1 chunk if maxChunkCount is not even + this.chunkBytesBuffer = new Uint8Array(chunkByteLen); + } + const chunkBytes = this.chunkBytesBuffer.subarray(0, chunkByteLen); + const uint8Array = chunkBytes.subarray(0, byteLen); const dataView = new DataView(uint8Array.buffer, uint8Array.byteOffset, uint8Array.byteLength); value_serializeToBytesArrayBasic(this.elementType, value.length, {uint8Array, dataView}, 0, value); - return splitIntoRootChunks(uint8Array); + + // all padding bytes must be zero, this is similar to set zeroHash(0) + this.chunkBytesBuffer.subarray(byteLen, chunkByteLen).fill(0); + return chunkBytes; } // JSON: inherited from ArrayType diff --git a/packages/ssz/src/type/listComposite.ts b/packages/ssz/src/type/listComposite.ts index dad8e77c..821b9504 100644 --- a/packages/ssz/src/type/listComposite.ts +++ b/packages/ssz/src/type/listComposite.ts @@ -1,10 +1,5 @@ -import {Node, Tree, HashComputationLevel} from "@chainsafe/persistent-merkle-tree"; -import { - mixInLength, - maxChunksToDepth, - symbolCachedPermanentRoot, - ValueWithCachedPermanentRoot, -} from "../util/merkleize"; +import {HashComputationLevel, Node, Tree, merkleizeInto} from "@chainsafe/persistent-merkle-tree"; +import {cacheRoot, maxChunksToDepth, symbolCachedPermanentRoot, ValueWithCachedPermanentRoot} from "../util/merkleize"; import {Require} from "../util/types"; import {namedClass} from "../util/named"; import {ValueOf, ByteViews} from "./abstract"; @@ -17,13 +12,14 @@ import { tree_serializedSizeArrayComposite, tree_deserializeFromBytesArrayComposite, tree_serializeToBytesArrayComposite, - value_getRootsArrayComposite, maxSizeArrayComposite, + value_getChunkBytesArrayComposite, } from "./arrayComposite"; import {ArrayCompositeType} from "../view/arrayComposite"; import {ListCompositeTreeView} from "../view/listComposite"; import {ListCompositeTreeViewDU} from "../viewDU/listComposite"; import {ArrayType} from "./array"; +import {allocUnsafe} from "@chainsafe/as-sha256"; /* eslint-disable @typescript-eslint/member-ordering */ @@ -56,6 +52,12 @@ export class ListCompositeType< readonly maxSize: number; readonly isList = true; readonly isViewMutable = true; + readonly mixInLengthChunkBytes = new Uint8Array(64); + readonly mixInLengthBuffer = Buffer.from( + this.mixInLengthChunkBytes.buffer, + this.mixInLengthChunkBytes.byteOffset, + this.mixInLengthChunkBytes.byteLength + ); protected readonly defaultLen = 0; constructor(readonly elementType: ElementType, readonly limit: number, opts?: ListCompositeOpts) { @@ -180,17 +182,43 @@ export class ListCompositeType< } } - const root = mixInLength(super.hashTreeRoot(value), value.length); + const root = allocUnsafe(32); + const safeCache = true; + this.hashTreeRootInto(value, root, 0, safeCache); + + // hashTreeRootInto will cache the root if cachePermanentRootStruct is true + return root; + } + + hashTreeRootInto(value: ValueOf[], output: Uint8Array, offset: number, safeCache = false): void { if (this.cachePermanentRootStruct) { - (value as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot] = root; + const cachedRoot = (value as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot]; + if (cachedRoot) { + output.set(cachedRoot, offset); + return; + } } - return root; + super.hashTreeRootInto(value, this.mixInLengthChunkBytes, 0); + // mixInLength + this.mixInLengthBuffer.writeUIntLE(value.length, 32, 6); + // one for hashTreeRoot(value), one for length + const chunkCount = 2; + merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset); + + if (this.cachePermanentRootStruct) { + cacheRoot(value as ValueWithCachedPermanentRoot, output, offset, safeCache); + } } - protected getRoots(value: ValueOf[]): Uint8Array[] { - return value_getRootsArrayComposite(this.elementType, value.length, value); + protected getChunkBytes(value: ValueOf[]): Uint8Array { + const byteLen = value.length * 32; + const chunkByteLen = this.chunkBytesBuffer.byteLength; + if (byteLen > chunkByteLen) { + this.chunkBytesBuffer = new Uint8Array(Math.ceil(byteLen / 64) * 64); + } + return value_getChunkBytesArrayComposite(this.elementType, value.length, value, this.chunkBytesBuffer); } // JSON: inherited from ArrayType diff --git a/packages/ssz/src/type/optional.ts b/packages/ssz/src/type/optional.ts index 7c5f9baf..3d2925fb 100644 --- a/packages/ssz/src/type/optional.ts +++ b/packages/ssz/src/type/optional.ts @@ -1,18 +1,19 @@ import { concatGindices, Gindex, + merkleizeInto, Node, Tree, zeroNode, - HashComputationLevel, getHashComputations, + HashComputationLevel, } from "@chainsafe/persistent-merkle-tree"; -import {mixInLength} from "../util/merkleize"; import {Require} from "../util/types"; import {namedClass} from "../util/named"; import {Type, ByteViews, JsonPath, JsonPathProp} from "./abstract"; import {CompositeType, isCompositeType} from "./composite"; import {addLengthNode, getLengthFromRootNode} from "./arrayBasic"; +import {allocUnsafe} from "@chainsafe/as-sha256"; /* eslint-disable @typescript-eslint/member-ordering */ export type NonOptionalType> = T extends OptionalType ? U : T; @@ -47,6 +48,12 @@ export class OptionalType> extends CompositeTy readonly maxSize: number; readonly isList = true; readonly isViewMutable = true; + readonly mixInLengthChunkBytes = new Uint8Array(64); + readonly mixInLengthBuffer = Buffer.from( + this.mixInLengthChunkBytes.buffer, + this.mixInLengthChunkBytes.byteOffset, + this.mixInLengthChunkBytes.byteLength + ); constructor(readonly elementType: ElementType, opts?: OptionalOpts) { super(); @@ -59,6 +66,7 @@ export class OptionalType> extends CompositeTy this.minSize = 0; // Max size includes prepended 0x01 byte this.maxSize = elementType.maxSize + 1; + this.chunkBytesBuffer = new Uint8Array(32); } static named>( @@ -171,13 +179,27 @@ export class OptionalType> extends CompositeTy // Merkleization hashTreeRoot(value: ValueOfType): Uint8Array { + const root = allocUnsafe(32); + this.hashTreeRootInto(value, root, 0); + return root; + } + + hashTreeRootInto(value: ValueOfType, output: Uint8Array, offset: number): void { + super.hashTreeRootInto(value, this.mixInLengthChunkBytes, 0); const selector = value === null ? 0 : 1; - return mixInLength(super.hashTreeRoot(value), selector); + this.mixInLengthBuffer.writeUIntLE(selector, 32, 6); + // one for hashTreeRoot(value), one for selector + const chunkCount = 2; + merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset); } - protected getRoots(value: ValueOfType): Uint8Array[] { - const valueRoot = value === null ? new Uint8Array(32) : this.elementType.hashTreeRoot(value); - return [valueRoot]; + protected getChunkBytes(value: ValueOfType): Uint8Array { + if (value === null) { + this.chunkBytesBuffer.fill(0); + } else { + this.elementType.hashTreeRootInto(value, this.chunkBytesBuffer, 0); + } + return this.chunkBytesBuffer; } // Proofs diff --git a/packages/ssz/src/type/profile.ts b/packages/ssz/src/type/profile.ts index f9469fe0..1ac440d7 100644 --- a/packages/ssz/src/type/profile.ts +++ b/packages/ssz/src/type/profile.ts @@ -6,12 +6,13 @@ import { Gindex, toGindex, concatGindices, + merkleizeInto, getNode, BranchNode, zeroHash, zeroNode, } from "@chainsafe/persistent-merkle-tree"; -import {ValueWithCachedPermanentRoot, maxChunksToDepth, symbolCachedPermanentRoot} from "../util/merkleize"; +import {ValueWithCachedPermanentRoot, cacheRoot, maxChunksToDepth, symbolCachedPermanentRoot} from "../util/merkleize"; import {Require} from "../util/types"; import {namedClass} from "../util/named"; import {Type, ValueOf} from "./abstract"; @@ -87,6 +88,8 @@ export class ProfileType>> extends C protected readonly TreeView: ContainerTreeViewTypeConstructor; protected readonly TreeViewDU: ContainerTreeViewDUTypeConstructor; private optionalFieldsCount: number; + // temporary root to avoid memory allocation + private tempRoot = new Uint8Array(32); constructor(readonly fields: Fields, activeFields: BitArray, readonly opts?: ProfileOptions) { super(); @@ -154,6 +157,9 @@ export class ProfileType>> extends C // Refactor this constructor to allow customization without pollutin the options this.TreeView = opts?.getProfileTreeViewClass?.(this) ?? getProfileTreeViewClass(this); this.TreeViewDU = opts?.getProfileTreeViewDUClass?.(this) ?? getProfileTreeViewDUClass(this); + const fieldBytes = this.activeFields.bitLen * 32; + const chunkBytes = Math.ceil(fieldBytes / 64) * 64; + this.chunkBytesBuffer = new Uint8Array(chunkBytes); } static named>>( @@ -361,37 +367,38 @@ export class ProfileType>> extends C } // Merkleization - hashTreeRoot(value: ValueOfFields): Uint8Array { + // hashTreeRoot is the same to parent as it call hashTreeRootInto() + hashTreeRootInto(value: ValueOfFields, output: Uint8Array, offset: number, safeCache = false): void { // Return cached mutable root if any if (this.cachePermanentRootStruct) { const cachedRoot = (value as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot]; if (cachedRoot) { - return cachedRoot; + output.set(cachedRoot, offset); + return; } } - const root = mixInActiveFields(super.hashTreeRoot(value), this.activeFields); + const merkleBytes = this.getChunkBytes(value); + merkleizeInto(merkleBytes, this.maxChunkCount, this.tempRoot, 0); + mixInActiveFields(this.tempRoot, this.activeFields, output, offset); if (this.cachePermanentRootStruct) { - (value as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot] = root; + cacheRoot(value as ValueWithCachedPermanentRoot, output, offset, safeCache); } - - return root; } - protected getRoots(struct: ValueOfFields): Uint8Array[] { - const roots = new Array(this.activeFields.bitLen).fill(zeroHash(0)); - - // already asserted that # of active fields in bitvector === # of fields + protected getChunkBytes(struct: ValueOfFields): Uint8Array { + this.chunkBytesBuffer.fill(0); for (let i = 0; i < this.fieldsEntries.length; i++) { const {fieldName, fieldType, chunkIndex, optional} = this.fieldsEntries[i]; if (optional && struct[fieldName] == null) { - continue; + this.chunkBytesBuffer.set(zeroHash(0), chunkIndex * 32); + } else { + fieldType.hashTreeRootInto(struct[fieldName], this.chunkBytesBuffer, chunkIndex * 32); } - roots[chunkIndex] = fieldType.hashTreeRoot(struct[fieldName]); } - - return roots; + // remaining bytes are zeroed as we never write them + return this.chunkBytesBuffer; } // Proofs diff --git a/packages/ssz/src/type/stableContainer.ts b/packages/ssz/src/type/stableContainer.ts index bf8b94fa..35415ad8 100644 --- a/packages/ssz/src/type/stableContainer.ts +++ b/packages/ssz/src/type/stableContainer.ts @@ -11,19 +11,13 @@ import { getNode, zeroNode, zeroHash, + merkleizeInto, countToDepth, getNodeH, setNode, setNodeWithFn, } from "@chainsafe/persistent-merkle-tree"; -import { - ValueWithCachedPermanentRoot, - hash64, - maxChunksToDepth, - merkleize, - splitIntoRootChunks, - symbolCachedPermanentRoot, -} from "../util/merkleize"; +import {ValueWithCachedPermanentRoot, cacheRoot, maxChunksToDepth, symbolCachedPermanentRoot} from "../util/merkleize"; import {Require} from "../util/types"; import {namedClass} from "../util/named"; import {JsonPath, Type, ValueOf} from "./abstract"; @@ -99,6 +93,8 @@ export class StableContainerType>> e protected readonly TreeView: ContainerTreeViewTypeConstructor; protected readonly TreeViewDU: ContainerTreeViewDUTypeConstructor; private padActiveFields: boolean[]; + // temporary root to avoid memory allocation + private tempRoot = new Uint8Array(32); constructor(fields: Fields, readonly maxFields: number, readonly opts?: StableContainerOptions) { super(); @@ -153,6 +149,9 @@ export class StableContainerType>> e // Refactor this constructor to allow customization without pollutin the options this.TreeView = opts?.getContainerTreeViewClass?.(this) ?? getContainerTreeViewClass(this); this.TreeViewDU = opts?.getContainerTreeViewDUClass?.(this) ?? getContainerTreeViewDUClass(this); + const fieldBytes = this.fieldsEntries.length * 32; + const chunkBytes = Math.ceil(fieldBytes / 64) * 64; + this.chunkBytesBuffer = new Uint8Array(chunkBytes); } static named>>( @@ -341,43 +340,43 @@ export class StableContainerType>> e } // Merkleization - hashTreeRoot(value: ValueOfFields): Uint8Array { + // hashTreeRoot is the same to parent as it call hashTreeRootInto() + hashTreeRootInto(value: ValueOfFields, output: Uint8Array, offset: number, safeCache = false): void { // Return cached mutable root if any if (this.cachePermanentRootStruct) { const cachedRoot = (value as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot]; if (cachedRoot) { - return cachedRoot; + output.set(cachedRoot, offset); + return; } } + const merkleBytes = this.getChunkBytes(value); + merkleizeInto(merkleBytes, this.maxChunkCount, this.tempRoot, 0); // compute active field bitvector const activeFields = BitArray.fromBoolArray([ ...this.fieldsEntries.map(({fieldName}) => value[fieldName] != null), ...this.padActiveFields, ]); - const root = mixInActiveFields(super.hashTreeRoot(value), activeFields); + mixInActiveFields(this.tempRoot, activeFields, output, offset); if (this.cachePermanentRootStruct) { - (value as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot] = root; + cacheRoot(value as ValueWithCachedPermanentRoot, output, offset, safeCache); } - - return root; } - protected getRoots(struct: ValueOfFields): Uint8Array[] { - const roots = new Array(this.fieldsEntries.length); - + protected getChunkBytes(struct: ValueOfFields): Uint8Array { + this.chunkBytesBuffer.fill(0); for (let i = 0; i < this.fieldsEntries.length; i++) { const {fieldName, fieldType, optional} = this.fieldsEntries[i]; if (optional && struct[fieldName] == null) { - roots[i] = zeroHash(0); - continue; + this.chunkBytesBuffer.set(zeroHash(0), i * 32); + } else { + fieldType.hashTreeRootInto(struct[fieldName], this.chunkBytesBuffer, i * 32); } - - roots[i] = fieldType.hashTreeRoot(struct[fieldName]); } - return roots; + return this.chunkBytesBuffer; } // Proofs @@ -751,12 +750,15 @@ export function getActiveFields(rootNode: Node, bitLen: number): BitArray { return new BitArray(activeFieldsBuf, bitLen); } +// This is a global buffer to avoid creating a new one for each call to getActiveFields +const singleChunkActiveFieldsBuf = new Uint8Array(32); + export function setActiveFields(rootNode: Node, activeFields: BitArray): Node { // fast path for depth 1, the bitvector fits in one chunk if (activeFields.bitLen <= 256) { - const activeFieldsBuf = new Uint8Array(32); - activeFieldsBuf.set(activeFields.uint8Array); - return new BranchNode(rootNode.left, LeafNode.fromRoot(activeFieldsBuf)); + singleChunkActiveFieldsBuf.fill(0); + singleChunkActiveFieldsBuf.set(activeFields.uint8Array); + return new BranchNode(rootNode.left, LeafNode.fromRoot(singleChunkActiveFieldsBuf)); } const activeFieldsChunkCount = Math.ceil(activeFields.bitLen / 256); @@ -815,15 +817,24 @@ export function setActiveField(rootNode: Node, bitLen: number, fieldIndex: numbe return new BranchNode(rootNode.left, newActiveFieldsNode); } -export function mixInActiveFields(root: Uint8Array, activeFields: BitArray): Uint8Array { +// This is a global buffer to avoid creating a new one for each call to getChunkBytes +const mixInActiveFieldsChunkBytes = new Uint8Array(64); +const activeFieldsSingleChunk = mixInActiveFieldsChunkBytes.subarray(32); + +export function mixInActiveFields(root: Uint8Array, activeFields: BitArray, output: Uint8Array, offset: number): void { // fast path for depth 1, the bitvector fits in one chunk + mixInActiveFieldsChunkBytes.set(root, 0); if (activeFields.bitLen <= 256) { - const activeFieldsChunk = new Uint8Array(32); - activeFieldsChunk.set(activeFields.uint8Array); - return hash64(root, activeFieldsChunk); - } - - const activeFieldsChunks = splitIntoRootChunks(activeFields.uint8Array); - const activeFieldsRoot = merkleize(activeFieldsChunks, activeFieldsChunks.length); - return hash64(root, activeFieldsRoot); + activeFieldsSingleChunk.fill(0); + activeFieldsSingleChunk.set(activeFields.uint8Array); + // 1 chunk for root, 1 chunk for activeFields + const chunkCount = 2; + merkleizeInto(mixInActiveFieldsChunkBytes, chunkCount, output, offset); + return; + } + + const chunkCount = Math.ceil(activeFields.uint8Array.length / 32); + merkleizeInto(activeFields.uint8Array, chunkCount, activeFieldsSingleChunk, 0); + // 1 chunk for root, 1 chunk for activeFields + merkleizeInto(mixInActiveFieldsChunkBytes, 2, output, offset); } diff --git a/packages/ssz/src/type/uint.ts b/packages/ssz/src/type/uint.ts index 910310f4..81b47a18 100644 --- a/packages/ssz/src/type/uint.ts +++ b/packages/ssz/src/type/uint.ts @@ -133,6 +133,12 @@ export class UintNumberType extends BasicType { } } + value_toTree(value: number): Node { + const node = LeafNode.fromZero(); + node.setUint(this.byteLength, 0, value, this.clipInfinity); + return node; + } + tree_serializeToBytes(output: ByteViews, offset: number, node: Node): number { const value = (node as LeafNode).getUint(this.byteLength, 0, this.clipInfinity); this.value_serializeToBytes(output, offset, value); diff --git a/packages/ssz/src/type/union.ts b/packages/ssz/src/type/union.ts index fbd7f97a..6a6117dd 100644 --- a/packages/ssz/src/type/union.ts +++ b/packages/ssz/src/type/union.ts @@ -4,16 +4,17 @@ import { Gindex, Node, Tree, - HashComputationLevel, + merkleizeInto, getHashComputations, + HashComputationLevel, } from "@chainsafe/persistent-merkle-tree"; -import {mixInLength} from "../util/merkleize"; import {Require} from "../util/types"; import {namedClass} from "../util/named"; import {Type, ByteViews} from "./abstract"; import {CompositeType, isCompositeType} from "./composite"; import {addLengthNode, getLengthFromRootNode} from "./arrayBasic"; import {NoneType} from "./none"; +import {allocUnsafe} from "@chainsafe/as-sha256"; /* eslint-disable @typescript-eslint/member-ordering */ @@ -48,6 +49,12 @@ export class UnionType[]> extends CompositeType< readonly maxSize: number; readonly isList = true; readonly isViewMutable = true; + readonly mixInLengthChunkBytes = new Uint8Array(64); + readonly mixInLengthBuffer = Buffer.from( + this.mixInLengthChunkBytes.buffer, + this.mixInLengthChunkBytes.byteOffset, + this.mixInLengthChunkBytes.byteLength + ); protected readonly maxSelector: number; @@ -85,6 +92,7 @@ export class UnionType[]> extends CompositeType< this.minSize = 1 + Math.min(...minLens); this.maxSize = 1 + Math.max(...maxLens); this.maxSelector = this.types.length - 1; + this.chunkBytesBuffer = new Uint8Array(32); } static named[]>(types: Types, opts: Require): UnionType { @@ -170,12 +178,21 @@ export class UnionType[]> extends CompositeType< // Merkleization hashTreeRoot(value: ValueOfTypes): Uint8Array { - return mixInLength(super.hashTreeRoot(value), value.selector); + const root = allocUnsafe(32); + this.hashTreeRootInto(value, root, 0); + return root; + } + + hashTreeRootInto(value: ValueOfTypes, output: Uint8Array, offset: number): void { + super.hashTreeRootInto(value, this.mixInLengthChunkBytes, 0); + this.mixInLengthBuffer.writeUIntLE(value.selector, 32, 6); + const chunkCount = 2; + merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset); } - protected getRoots(value: ValueOfTypes): Uint8Array[] { - const valueRoot = this.types[value.selector].hashTreeRoot(value.value); - return [valueRoot]; + protected getChunkBytes(value: ValueOfTypes): Uint8Array { + this.types[value.selector].hashTreeRootInto(value.value, this.chunkBytesBuffer, 0); + return this.chunkBytesBuffer; } // Proofs diff --git a/packages/ssz/src/type/vectorBasic.ts b/packages/ssz/src/type/vectorBasic.ts index d52a9405..bb189044 100644 --- a/packages/ssz/src/type/vectorBasic.ts +++ b/packages/ssz/src/type/vectorBasic.ts @@ -1,5 +1,5 @@ -import {Node, Tree, HashComputationLevel} from "@chainsafe/persistent-merkle-tree"; -import {maxChunksToDepth, splitIntoRootChunks} from "../util/merkleize"; +import {HashComputationLevel, Node, Tree} from "@chainsafe/persistent-merkle-tree"; +import {maxChunksToDepth} from "../util/merkleize"; import {Require} from "../util/types"; import {namedClass} from "../util/named"; import {ValueOf, ByteViews} from "./abstract"; @@ -59,6 +59,10 @@ export class VectorBasicType> this.minSize = this.fixedSize; this.maxSize = this.fixedSize; this.defaultLen = length; + // pad 1 chunk if maxChunkCount is not even + this.chunkBytesBuffer = new Uint8Array( + this.maxChunkCount % 2 === 1 ? this.maxChunkCount * 32 + 32 : this.maxChunkCount * 32 + ); } static named>( @@ -146,11 +150,13 @@ export class VectorBasicType> // Merkleization - protected getRoots(value: ValueOf[]): Uint8Array[] { - const uint8Array = new Uint8Array(this.fixedSize); + protected getChunkBytes(value: ValueOf[]): Uint8Array { + const uint8Array = this.chunkBytesBuffer.subarray(0, this.fixedSize); const dataView = new DataView(uint8Array.buffer, uint8Array.byteOffset, uint8Array.byteLength); value_serializeToBytesArrayBasic(this.elementType, this.length, {uint8Array, dataView}, 0, value); - return splitIntoRootChunks(uint8Array); + + // remaining bytes from this.fixedSize to this.chunkBytesBuffer.length must be zeroed + return this.chunkBytesBuffer; } // JSON: inherited from ArrayType diff --git a/packages/ssz/src/type/vectorComposite.ts b/packages/ssz/src/type/vectorComposite.ts index e1af8dd4..28990c43 100644 --- a/packages/ssz/src/type/vectorComposite.ts +++ b/packages/ssz/src/type/vectorComposite.ts @@ -11,9 +11,9 @@ import { tree_serializedSizeArrayComposite, tree_deserializeFromBytesArrayComposite, tree_serializeToBytesArrayComposite, - value_getRootsArrayComposite, maxSizeArrayComposite, minSizeArrayComposite, + value_getChunkBytesArrayComposite, } from "./arrayComposite"; import {ArrayCompositeType, ArrayCompositeTreeView} from "../view/arrayComposite"; import {ArrayCompositeTreeViewDU} from "../viewDU/arrayComposite"; @@ -65,6 +65,10 @@ export class VectorCompositeType< this.minSize = minSizeArrayComposite(elementType, length); this.maxSize = maxSizeArrayComposite(elementType, length); this.defaultLen = length; + this.chunkBytesBuffer = + this.maxChunkCount % 2 === 1 + ? new Uint8Array(this.maxChunkCount * 32 + 32) + : new Uint8Array(this.maxChunkCount * 32); } // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -153,8 +157,8 @@ export class VectorCompositeType< // Merkleization - protected getRoots(value: ValueOf[]): Uint8Array[] { - return value_getRootsArrayComposite(this.elementType, this.length, value); + protected getChunkBytes(value: ValueOf[]): Uint8Array { + return value_getChunkBytesArrayComposite(this.elementType, this.length, value, this.chunkBytesBuffer); } // JSON: inherited from ArrayType diff --git a/packages/ssz/src/util/merkleize.ts b/packages/ssz/src/util/merkleize.ts index 073dea5d..932e80d7 100644 --- a/packages/ssz/src/util/merkleize.ts +++ b/packages/ssz/src/util/merkleize.ts @@ -1,5 +1,4 @@ -import {hasher} from "@chainsafe/persistent-merkle-tree/lib/hasher/index"; -import {zeroHash} from "@chainsafe/persistent-merkle-tree"; +import {hasher, zeroHash} from "@chainsafe/persistent-merkle-tree"; /** Dedicated property to cache hashTreeRoot of immutable CompositeType values */ export const symbolCachedPermanentRoot = Symbol("ssz_cached_permanent_root"); @@ -9,6 +8,28 @@ export type ValueWithCachedPermanentRoot = { [symbolCachedPermanentRoot]?: Uint8Array; }; +/** + * Cache a root for a ValueWithCachedPermanentRoot instance + * - if safeCache is true and output is 32 bytes and offset is 0, use output directly + * - if safeCache, use output subarray + * - otherwise, need to clone the root at output offset + */ +export function cacheRoot( + value: ValueWithCachedPermanentRoot, + output: Uint8Array, + offset: number, + safeCache: boolean +): void { + const cachedRoot = + safeCache && output.length === 32 && offset === 0 + ? output + : safeCache + ? output.subarray(offset, offset + 32) + : // Buffer.prototype.slice does not copy memory, Enforce Uint8Array usage https://github.com/nodejs/node/issues/28087 + Uint8Array.prototype.slice.call(output, offset, offset + 32); + value[symbolCachedPermanentRoot] = cachedRoot; +} + export function hash64(bytes32A: Uint8Array, bytes32B: Uint8Array): Uint8Array { return hasher.digest64(bytes32A, bytes32B); } @@ -43,22 +64,6 @@ export function merkleize(chunks: Uint8Array[], padFor: number): Uint8Array { return chunks[0]; } -/** - * Split a long Uint8Array into Uint8Array of exactly 32 bytes - */ -export function splitIntoRootChunks(longChunk: Uint8Array): Uint8Array[] { - const chunkCount = Math.ceil(longChunk.length / 32); - const chunks = new Array(chunkCount); - - for (let i = 0; i < chunkCount; i++) { - const chunk = new Uint8Array(32); - chunk.set(longChunk.slice(i * 32, (i + 1) * 32)); - chunks[i] = chunk; - } - - return chunks; -} - /** @ignore */ export function mixInLength(root: Uint8Array, length: number): Uint8Array { const lengthBuf = Buffer.alloc(32); diff --git a/packages/ssz/test/perf/eth2/beaconBlock.test.ts b/packages/ssz/test/perf/eth2/beaconBlock.test.ts new file mode 100644 index 00000000..523e94ea --- /dev/null +++ b/packages/ssz/test/perf/eth2/beaconBlock.test.ts @@ -0,0 +1,94 @@ +import {itBench, setBenchOpts} from "@dapplion/benchmark"; +import {ValueWithCachedPermanentRoot, symbolCachedPermanentRoot} from "../../../src/util/merkleize"; +import {deneb, ssz} from "../../lodestarTypes"; +import {preset} from "../../lodestarTypes/params"; +import {BitArray, toHexString} from "../../../src"; +const {MAX_ATTESTATIONS, MAX_DEPOSITS, MAX_VOLUNTARY_EXITS, MAX_BLS_TO_EXECUTION_CHANGES} = preset; + +describe("Benchmark BeaconBlock.hashTreeRoot()", function () { + setBenchOpts({ + minMs: 10_000, + }); + + const block = ssz.deneb.BeaconBlock.defaultValue(); + for (let i = 0; i < MAX_ATTESTATIONS; i++) { + block.body.attestations.push({ + aggregationBits: BitArray.fromBoolArray(Array.from({length: 64}, () => true)), + data: { + slot: 1, + index: 1, + beaconBlockRoot: Buffer.alloc(32, 1), + source: { + epoch: 1, + root: Buffer.alloc(32, 1), + }, + target: { + epoch: 1, + root: Buffer.alloc(32, 1), + }, + }, + signature: Buffer.alloc(96, 1), + }); + } + for (let i = 0; i < MAX_DEPOSITS; i++) { + block.body.deposits.push({ + proof: ssz.phase0.Deposit.fields.proof.defaultValue(), + data: { + pubkey: Buffer.alloc(48, 1), + withdrawalCredentials: Buffer.alloc(32, 1), + amount: 32 * 1e9, + signature: Buffer.alloc(96, 1), + }, + }); + } + for (let i = 0; i < MAX_VOLUNTARY_EXITS; i++) { + block.body.voluntaryExits.push({ + signature: Buffer.alloc(96, 1), + message: { + epoch: 1, + validatorIndex: 1, + }, + }); + } + // common data on mainnet as of Jun 2024 + const numTransaction = 200; + const transactionLen = 500; + for (let i = 0; i < numTransaction; i++) { + block.body.executionPayload.transactions.push(Buffer.alloc(transactionLen, 1)); + } + for (let i = 0; i < MAX_BLS_TO_EXECUTION_CHANGES; i++) { + block.body.blsToExecutionChanges.push({ + signature: Buffer.alloc(96, 1), + message: { + validatorIndex: 1, + fromBlsPubkey: Buffer.alloc(48, 1), + toExecutionAddress: Buffer.alloc(20, 1), + }, + }); + } + + const root = ssz.deneb.BeaconBlock.hashTreeRoot(block); + console.log("BeaconBlock.hashTreeRoot() root", toHexString(root)); + itBench({ + id: `Deneb BeaconBlock.hashTreeRoot(), numTransaction=${numTransaction}`, + beforeEach: () => { + clearCachedRoots(block); + return block; + }, + fn: (block: deneb.BeaconBlock) => { + ssz.deneb.BeaconBlock.hashTreeRoot(block); + }, + }); +}); + +function clearCachedRoots(block: deneb.BeaconBlock): void { + (block as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot] = undefined; + (block.body as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot] = undefined; + const attestations = block.body.attestations; + for (const attestation of attestations) { + (attestation.data as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot] = undefined; + } + for (const exit of block.body.voluntaryExits) { + (exit as ValueWithCachedPermanentRoot)[symbolCachedPermanentRoot] = undefined; + } +} diff --git a/packages/ssz/test/perf/merkleize.test.ts b/packages/ssz/test/perf/merkleize.test.ts index b83a1f5d..a900015a 100644 --- a/packages/ssz/test/perf/merkleize.test.ts +++ b/packages/ssz/test/perf/merkleize.test.ts @@ -1,5 +1,6 @@ import {itBench} from "@dapplion/benchmark"; -import {bitLength} from "../../src/util/merkleize"; +import {bitLength, merkleize} from "../../src/util/merkleize"; +import {merkleizeInto} from "@chainsafe/persistent-merkle-tree"; describe("merkleize / bitLength", () => { for (const n of [50, 8000, 250000]) { @@ -13,6 +14,23 @@ describe("merkleize / bitLength", () => { } }); +describe("merkleize vs persistent-merkle-tree merkleizeInto", () => { + const chunkCounts = [4, 8, 16, 32]; + + for (const chunkCount of chunkCounts) { + const rootArr = Array.from({length: chunkCount}, (_, i) => Buffer.alloc(32, i)); + const roots = Buffer.concat(rootArr); + const result = Buffer.alloc(32); + itBench(`merkleizeInto ${chunkCount} chunks`, () => { + merkleizeInto(roots, chunkCount, result, 0); + }); + + itBench(`merkleize ${chunkCount} chunks`, () => { + merkleize(rootArr, chunkCount); + }); + } +}); + // Previous implementation, replaced by bitLength function bitLengthStr(n: number): number { const bitstring = n.toString(2); diff --git a/packages/ssz/test/spec/runValidTest.ts b/packages/ssz/test/spec/runValidTest.ts index 1bac7760..5ea219eb 100644 --- a/packages/ssz/test/spec/runValidTest.ts +++ b/packages/ssz/test/spec/runValidTest.ts @@ -101,13 +101,10 @@ export function runValidSszTest(type: Type, testData: ValidTestCaseData // 0x0000000000000000000000000000000000000000000000000000000000000000 if (process.env.RENDER_ROOTS) { if (type.isBasic) { - console.log("ROOTS Basic", toHexString(type.serialize(testDataValue))); + console.log("Chunk Bytes Basic", toHexString(type.serialize(testDataValue))); } else { - const roots = (type as CompositeType)["getRoots"](testDataValue); - console.log( - "ROOTS Composite", - roots.map((root) => toHexString(root)) - ); + const chunkBytes = (type as CompositeType)["getChunkBytes"](testDataValue); + console.log("Chunk Bytes Composite", toHexString(chunkBytes)); } } diff --git a/packages/ssz/test/unit/merkleize.test.ts b/packages/ssz/test/unit/merkleize.test.ts index 6b996c8c..d1e611b7 100644 --- a/packages/ssz/test/unit/merkleize.test.ts +++ b/packages/ssz/test/unit/merkleize.test.ts @@ -1,5 +1,6 @@ import {expect} from "chai"; -import {bitLength, maxChunksToDepth, nextPowerOf2} from "../../src/util/merkleize"; +import {bitLength, maxChunksToDepth, merkleize, mixInLength, nextPowerOf2} from "../../src/util/merkleize"; +import {merkleizeInto, LeafNode, zeroHash} from "@chainsafe/persistent-merkle-tree"; describe("util / merkleize / bitLength", () => { const bitLengthByIndex = [0, 1, 2, 2, 3, 3, 3, 3, 4, 4]; @@ -30,3 +31,35 @@ describe("util / merkleize / nextPowerOf2", () => { }); } }); + +describe("util / merkleize / mixInLength", () => { + const root = Buffer.alloc(32, 1); + const lengths = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; + for (const length of lengths) { + it(`mixInLength(${length})`, () => { + const mixInLengthBuffer = Buffer.alloc(64); + mixInLengthBuffer.set(root, 0); + mixInLengthBuffer.writeUIntLE(length, 32, 6); + const finalRoot = new Uint8Array(32); + merkleizeInto(mixInLengthBuffer, 2, finalRoot, 0); + const expectedRoot = mixInLength(root, length); + expect(finalRoot).to.be.deep.equal(expectedRoot); + }); + } +}); + +describe("merkleize should be equal to merkleizeInto of hasher", () => { + const numNodes = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + for (const numNode of numNodes) { + it(`merkleize for ${numNode} nodes`, () => { + const nodes = Array.from({length: numNode}, (_, i) => LeafNode.fromRoot(Buffer.alloc(32, i))); + const data = Buffer.concat(nodes.map((node) => node.root)); + const padData = numNode % 2 === 1 ? Buffer.concat([data, zeroHash(0)]) : data; + const roots = nodes.map((node) => node.root); + const expectedRoot = Buffer.alloc(32); + const chunkCount = Math.max(numNode, 1); + merkleizeInto(padData, chunkCount, expectedRoot, 0); + expect(merkleize(roots, chunkCount)).to.be.deep.equal(expectedRoot); + }); + } +});