Skip to content

Commit

Permalink
refactor: merkleizeInto to merkleizeBlocksBytes
Browse files Browse the repository at this point in the history
  • Loading branch information
twoeths committed Oct 31, 2024
1 parent 3249906 commit 025a069
Show file tree
Hide file tree
Showing 18 changed files with 52 additions and 47 deletions.
6 changes: 3 additions & 3 deletions packages/persistent-merkle-tree/src/hasher/as-sha256.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,14 @@ import {
import type {Hasher} from "./types";
import {Node} from "../node";
import type {HashComputationLevel} from "../hashComputation";
import {doDigestNLevel, doMerkleizeInto} from "./util";
import {doDigestNLevel, doMerkleizeBlocksBytes} from "./util";

export const hasher: Hasher = {
name: "as-sha256",
digest64: digest2Bytes32,
digest64HashObjects: digest64HashObjectsInto,
merkleizeInto(blocksBytes: Uint8Array, padFor: number, output: Uint8Array, offset: number): void {
return doMerkleizeInto(blocksBytes, padFor, output, offset, hashInto);
merkleizeBlocksBytes(blocksBytes: Uint8Array, padFor: number, output: Uint8Array, offset: number): void {
return doMerkleizeBlocksBytes(blocksBytes, padFor, output, offset, hashInto);
},
digestNLevel(data: Uint8Array, nLevel: number): Uint8Array {
return doDigestNLevel(data, nLevel, hashInto);
Expand Down
6 changes: 3 additions & 3 deletions packages/persistent-merkle-tree/src/hasher/hashtree.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import {Hasher, HashObject} from "./types";
import {Node} from "../node";
import type {HashComputationLevel} from "../hashComputation";
import {byteArrayIntoHashObject} from "@chainsafe/as-sha256/lib/hashObject";
import {doDigestNLevel, doMerkleizeInto} from "./util";
import {doDigestNLevel, doMerkleizeBlocksBytes} from "./util";

/**
* Best SIMD implementation is in 512 bits = 64 bytes
Expand Down Expand Up @@ -40,8 +40,8 @@ export const hasher: Hasher = {
hashInto(hash64Input, hash64Output);
byteArrayIntoHashObject(hash64Output, 0, parent);
},
merkleizeInto(blocksBytes: Uint8Array, padFor: number, output: Uint8Array, offset: number): void {
return doMerkleizeInto(blocksBytes, padFor, output, offset, hashInto);
merkleizeBlocksBytes(blocksBytes: Uint8Array, padFor: number, output: Uint8Array, offset: number): void {
return doMerkleizeBlocksBytes(blocksBytes, padFor, output, offset, hashInto);
},
digestNLevel(data: Uint8Array, nLevel: number): Uint8Array {
return doDigestNLevel(data, nLevel, hashInto);
Expand Down
9 changes: 7 additions & 2 deletions packages/persistent-merkle-tree/src/hasher/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,13 @@ export function digestNLevel(data: Uint8Array, nLevel: number): Uint8Array {
return hasher.digestNLevel(data, nLevel);
}

export function merkleizeInto(blocksBytes: Uint8Array, padFor: number, output: Uint8Array, offset: number): void {
hasher.merkleizeInto(blocksBytes, padFor, output, offset);
export function merkleizeBlocksBytes(
blocksBytes: Uint8Array,
padFor: number,
output: Uint8Array,
offset: number
): void {
hasher.merkleizeBlocksBytes(blocksBytes, padFor, output, offset);
}

export function executeHashComputations(hashComputations: HashComputationLevel[]): void {
Expand Down
6 changes: 3 additions & 3 deletions packages/persistent-merkle-tree/src/hasher/noble.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import {sha256} from "@noble/hashes/sha256";
import {digest64HashObjects, byteArrayIntoHashObject} from "@chainsafe/as-sha256";
import type {Hasher} from "./types";
import {doDigestNLevel, doMerkleizeInto, hashObjectToUint8Array} from "./util";
import {doDigestNLevel, doMerkleizeBlocksBytes, hashObjectToUint8Array} from "./util";

const digest64 = (a: Uint8Array, b: Uint8Array): Uint8Array => sha256.create().update(a).update(b).digest();
const hashInto = (input: Uint8Array, output: Uint8Array): void => {
Expand All @@ -28,8 +28,8 @@ export const hasher: Hasher = {
digest64HashObjects: (left, right, parent) => {
byteArrayIntoHashObject(digest64(hashObjectToUint8Array(left), hashObjectToUint8Array(right)), 0, parent);
},
merkleizeInto(blocksBytes: Uint8Array, padFor: number, output: Uint8Array, offset: number): void {
return doMerkleizeInto(blocksBytes, padFor, output, offset, hashInto);
merkleizeBlocksBytes(blocksBytes: Uint8Array, padFor: number, output: Uint8Array, offset: number): void {
return doMerkleizeBlocksBytes(blocksBytes, padFor, output, offset, hashInto);
},
digestNLevel(data: Uint8Array, nLevel: number): Uint8Array {
return doDigestNLevel(data, nLevel, hashInto);
Expand Down
4 changes: 2 additions & 2 deletions packages/persistent-merkle-tree/src/hasher/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,11 @@ export type Hasher = {
*/
digest64HashObjects(left: HashObject, right: HashObject, parent: HashObject): void;
/**
* Merkleize n SHA256 blocks, each block is 64 bytes
* Merkleize n SHA256 blocks in a single Uint8Array, each block is 64 bytes
* padFor is maxChunkCount, use it to compute layers to hash
* blocksBytes is mutated after the function
*/
merkleizeInto(blocksBytes: Uint8Array, padFor: number, output: Uint8Array, offset: number): void;
merkleizeBlocksBytes(blocksBytes: Uint8Array, padFor: number, output: Uint8Array, offset: number): void;
/**
* Hash multiple chunks (1 chunk = 32 bytes) at multiple levels
* With nLevel = 3, hash multiple of 256 bytes, return multiple of 32 bytes.
Expand Down
2 changes: 1 addition & 1 deletion packages/persistent-merkle-tree/src/hasher/util.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ type HashIntoFn = (input: Uint8Array, output: Uint8Array) => void;
* blocksBytes is unsafe because it's modified
* The Uint8Array(32) will be written to output at offset
*/
export function doMerkleizeInto(
export function doMerkleizeBlocksBytes(
blocksBytes: Uint8Array,
padFor: number,
output: Uint8Array,
Expand Down
8 changes: 4 additions & 4 deletions packages/persistent-merkle-tree/test/unit/hasher.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -89,24 +89,24 @@ describe("hasher.digestNLevel", function () {
});


describe("hasher.merkleizeInto", function () {
describe("hasher.merkleizeBlocksBytes", function () {
const numNodes = [0, 1, 2, 3, 4, 5, 6, 7, 8];
for (const hasher of [nobleHasher, hashtreeHasher, asSha256Hasher]) {
it (`${hasher.name} should throw error if not multiple of 64 bytes`, () => {
const data = Buffer.alloc(63, 0);
const output = Buffer.alloc(32);
expect(() => hasher.merkleizeInto(data, 2, output, 0)).to.throw("Invalid input length");
expect(() => hasher.merkleizeBlocksBytes(data, 2, output, 0)).to.throw("Invalid input length");
});

for (const numNode of numNodes) {
it(`${hasher.name}.merkleizeInto for ${numNode} nodes`, () => {
it(`${hasher.name}.merkleizeBlocksBytes for ${numNode} nodes`, () => {

const nodes = Array.from({length: numNode}, (_, i) => LeafNode.fromRoot(Buffer.alloc(32, i)));
const data = Buffer.concat(nodes.map((node) => node.root));
const output = Buffer.alloc(32);
const chunkCount = Math.max(numNode, 1);
const padData = numNode % 2 === 1 ? Buffer.concat([data, zeroHash(0)]) : data;
hasher.merkleizeInto(padData, chunkCount, output, 0);
hasher.merkleizeBlocksBytes(padData, chunkCount, output, 0);
const depth = Math.ceil(Math.log2(chunkCount));
const root = subtreeFillToContents(nodes, depth).root;
expectEqualHex(output, root);
Expand Down
4 changes: 2 additions & 2 deletions packages/ssz/src/type/bitList.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import {allocUnsafe} from "@chainsafe/as-sha256";
import {
getNodesAtDepth,
merkleizeInto,
merkleizeBlocksBytes,
Node,
packedNodeRootsToBytes,
packedRootsBytesToNode,
Expand Down Expand Up @@ -125,7 +125,7 @@ export class BitListType extends BitArrayType {
this.mixInLengthBuffer.writeUIntLE(value.bitLen, 32, 6);
// one for hashTreeRoot(value), one for length
const chunkCount = 2;
merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset);
merkleizeBlocksBytes(this.mixInLengthChunkBytes, chunkCount, output, offset);
}

// Proofs: inherited from BitArrayType
Expand Down
4 changes: 2 additions & 2 deletions packages/ssz/src/type/byteList.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import {
Node,
packedNodeRootsToBytes,
packedRootsBytesToNode,
merkleizeInto,
merkleizeBlocksBytes,
} from "@chainsafe/persistent-merkle-tree";
import {maxChunksToDepth} from "../util/merkleize";
import {Require} from "../util/types";
Expand Down Expand Up @@ -112,7 +112,7 @@ export class ByteListType extends ByteArrayType {
this.mixInLengthBuffer.writeUIntLE(value.length, 32, 6);
// one for hashTreeRoot(value), one for length
const chunkCount = 2;
merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset);
merkleizeBlocksBytes(this.mixInLengthChunkBytes, chunkCount, output, offset);
}

// Proofs: inherited from BitArrayType
Expand Down
4 changes: 2 additions & 2 deletions packages/ssz/src/type/composite.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import {
Proof,
ProofType,
Tree,
merkleizeInto,
merkleizeBlocksBytes,
HashComputationLevel,
} from "@chainsafe/persistent-merkle-tree";
import {byteArrayEquals} from "../util/byteArray";
Expand Down Expand Up @@ -239,7 +239,7 @@ export abstract class CompositeType<V, TV, TVDU> extends Type<V> {
}

const blocksBuffer = this.getBlocksBytes(value);
merkleizeInto(blocksBuffer, this.maxChunkCount, output, offset);
merkleizeBlocksBytes(blocksBuffer, this.maxChunkCount, output, offset);
if (this.cachePermanentRootStruct) {
cacheRoot(value as ValueWithCachedPermanentRoot, output, offset, safeCache);
}
Expand Down
4 changes: 2 additions & 2 deletions packages/ssz/src/type/listBasic.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import {HashComputationLevel, LeafNode, Node, Tree, merkleizeInto} from "@chainsafe/persistent-merkle-tree";
import {HashComputationLevel, LeafNode, Node, Tree, merkleizeBlocksBytes} from "@chainsafe/persistent-merkle-tree";
import {ValueOf} from "./abstract";
import {BasicType} from "./basic";
import {ByteViews} from "./composite";
Expand Down Expand Up @@ -198,7 +198,7 @@ export class ListBasicType<ElementType extends BasicType<unknown>>
this.mixInLengthBuffer.writeUIntLE(value.length, 32, 6);
// one for hashTreeRoot(value), one for length
const chunkCount = 2;
merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset);
merkleizeBlocksBytes(this.mixInLengthChunkBytes, chunkCount, output, offset);

if (this.cachePermanentRootStruct) {
cacheRoot(value as ValueWithCachedPermanentRoot, output, offset, safeCache);
Expand Down
4 changes: 2 additions & 2 deletions packages/ssz/src/type/listComposite.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import {HashComputationLevel, Node, Tree, merkleizeInto} from "@chainsafe/persistent-merkle-tree";
import {HashComputationLevel, Node, Tree, merkleizeBlocksBytes} from "@chainsafe/persistent-merkle-tree";
import {cacheRoot, maxChunksToDepth, symbolCachedPermanentRoot, ValueWithCachedPermanentRoot} from "../util/merkleize";
import {Require} from "../util/types";
import {namedClass} from "../util/named";
Expand Down Expand Up @@ -205,7 +205,7 @@ export class ListCompositeType<
this.mixInLengthBuffer.writeUIntLE(value.length, 32, 6);
// one for hashTreeRoot(value), one for length
const chunkCount = 2;
merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset);
merkleizeBlocksBytes(this.mixInLengthChunkBytes, chunkCount, output, offset);

if (this.cachePermanentRootStruct) {
cacheRoot(value as ValueWithCachedPermanentRoot, output, offset, safeCache);
Expand Down
4 changes: 2 additions & 2 deletions packages/ssz/src/type/optional.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import {
concatGindices,
Gindex,
merkleizeInto,
merkleizeBlocksBytes,
Node,
Tree,
zeroNode,
Expand Down Expand Up @@ -190,7 +190,7 @@ export class OptionalType<ElementType extends Type<unknown>> extends CompositeTy
this.mixInLengthBuffer.writeUIntLE(selector, 32, 6);
// one for hashTreeRoot(value), one for selector
const chunkCount = 2;
merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset);
merkleizeBlocksBytes(this.mixInLengthChunkBytes, chunkCount, output, offset);
}

protected getBlocksBytes(value: ValueOfType<ElementType>): Uint8Array {
Expand Down
4 changes: 2 additions & 2 deletions packages/ssz/src/type/profile.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import {
Gindex,
toGindex,
concatGindices,
merkleizeInto,
merkleizeBlocksBytes,
getNode,
BranchNode,
zeroHash,
Expand Down Expand Up @@ -379,7 +379,7 @@ export class ProfileType<Fields extends Record<string, Type<unknown>>> extends C
}

const blocksBytes = this.getBlocksBytes(value);
merkleizeInto(blocksBytes, this.maxChunkCount, this.tempRoot, 0);
merkleizeBlocksBytes(blocksBytes, this.maxChunkCount, this.tempRoot, 0);
mixInActiveFields(this.tempRoot, this.activeFields, output, offset);

if (this.cachePermanentRootStruct) {
Expand Down
10 changes: 5 additions & 5 deletions packages/ssz/src/type/stableContainer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import {
getNode,
zeroNode,
zeroHash,
merkleizeInto,
merkleizeBlocksBytes,
countToDepth,
getNodeH,
setNode,
Expand Down Expand Up @@ -352,7 +352,7 @@ export class StableContainerType<Fields extends Record<string, Type<unknown>>> e
}

const blockBytes = this.getBlocksBytes(value);
merkleizeInto(blockBytes, this.maxChunkCount, this.tempRoot, 0);
merkleizeBlocksBytes(blockBytes, this.maxChunkCount, this.tempRoot, 0);
// compute active field bitvector
const activeFields = BitArray.fromBoolArray([
...this.fieldsEntries.map(({fieldName}) => value[fieldName] != null),
Expand Down Expand Up @@ -829,12 +829,12 @@ export function mixInActiveFields(root: Uint8Array, activeFields: BitArray, outp
activeFieldsSingleChunk.set(activeFields.uint8Array);
// 1 chunk for root, 1 chunk for activeFields
const chunkCount = 2;
merkleizeInto(mixInActiveFieldsChunkBytes, chunkCount, output, offset);
merkleizeBlocksBytes(mixInActiveFieldsChunkBytes, chunkCount, output, offset);
return;
}

const chunkCount = Math.ceil(activeFields.uint8Array.length / 32);
merkleizeInto(activeFields.uint8Array, chunkCount, activeFieldsSingleChunk, 0);
merkleizeBlocksBytes(activeFields.uint8Array, chunkCount, activeFieldsSingleChunk, 0);
// 1 chunk for root, 1 chunk for activeFields
merkleizeInto(mixInActiveFieldsChunkBytes, 2, output, offset);
merkleizeBlocksBytes(mixInActiveFieldsChunkBytes, 2, output, offset);
}
4 changes: 2 additions & 2 deletions packages/ssz/src/type/union.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import {
Gindex,
Node,
Tree,
merkleizeInto,
merkleizeBlocksBytes,
getHashComputations,
HashComputationLevel,
} from "@chainsafe/persistent-merkle-tree";
Expand Down Expand Up @@ -187,7 +187,7 @@ export class UnionType<Types extends Type<unknown>[]> extends CompositeType<
super.hashTreeRootInto(value, this.mixInLengthChunkBytes, 0);
this.mixInLengthBuffer.writeUIntLE(value.selector, 32, 6);
const chunkCount = 2;
merkleizeInto(this.mixInLengthChunkBytes, chunkCount, output, offset);
merkleizeBlocksBytes(this.mixInLengthChunkBytes, chunkCount, output, offset);
}

protected getBlocksBytes(value: ValueOfTypes<Types>): Uint8Array {
Expand Down
8 changes: 4 additions & 4 deletions packages/ssz/test/perf/merkleize.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import {itBench} from "@dapplion/benchmark";
import {bitLength, merkleize} from "../../src/util/merkleize";
import {merkleizeInto} from "@chainsafe/persistent-merkle-tree";
import {merkleizeBlocksBytes} from "@chainsafe/persistent-merkle-tree";

describe("merkleize / bitLength", () => {
for (const n of [50, 8000, 250000]) {
Expand All @@ -14,15 +14,15 @@ describe("merkleize / bitLength", () => {
}
});

describe("merkleize vs persistent-merkle-tree merkleizeInto", () => {
describe("merkleize vs persistent-merkle-tree merkleizeBlocksBytes", () => {
const chunkCounts = [4, 8, 16, 32];

for (const chunkCount of chunkCounts) {
const rootArr = Array.from({length: chunkCount}, (_, i) => Buffer.alloc(32, i));
const roots = Buffer.concat(rootArr);
const result = Buffer.alloc(32);
itBench(`merkleizeInto ${chunkCount} chunks`, () => {
merkleizeInto(roots, chunkCount, result, 0);
itBench(`merkleizeBlocksBytes ${chunkCount} chunks`, () => {
merkleizeBlocksBytes(roots, chunkCount, result, 0);
});

itBench(`merkleize ${chunkCount} chunks`, () => {
Expand Down
8 changes: 4 additions & 4 deletions packages/ssz/test/unit/merkleize.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import {expect} from "chai";
import {bitLength, maxChunksToDepth, merkleize, mixInLength, nextPowerOf2} from "../../src/util/merkleize";
import {merkleizeInto, LeafNode, zeroHash} from "@chainsafe/persistent-merkle-tree";
import {merkleizeBlocksBytes, LeafNode, zeroHash} from "@chainsafe/persistent-merkle-tree";

describe("util / merkleize / bitLength", () => {
const bitLengthByIndex = [0, 1, 2, 2, 3, 3, 3, 3, 4, 4];
Expand Down Expand Up @@ -41,14 +41,14 @@ describe("util / merkleize / mixInLength", () => {
mixInLengthBuffer.set(root, 0);
mixInLengthBuffer.writeUIntLE(length, 32, 6);
const finalRoot = new Uint8Array(32);
merkleizeInto(mixInLengthBuffer, 2, finalRoot, 0);
merkleizeBlocksBytes(mixInLengthBuffer, 2, finalRoot, 0);
const expectedRoot = mixInLength(root, length);
expect(finalRoot).to.be.deep.equal(expectedRoot);
});
}
});

describe("merkleize should be equal to merkleizeInto of hasher", () => {
describe("merkleize should be equal to merkleizeBlocksBytes of hasher", () => {
const numNodes = [0, 1, 2, 3, 4, 5, 6, 7, 8];
for (const numNode of numNodes) {
it(`merkleize for ${numNode} nodes`, () => {
Expand All @@ -58,7 +58,7 @@ describe("merkleize should be equal to merkleizeInto of hasher", () => {
const roots = nodes.map((node) => node.root);
const expectedRoot = Buffer.alloc(32);
const chunkCount = Math.max(numNode, 1);
merkleizeInto(padData, chunkCount, expectedRoot, 0);
merkleizeBlocksBytes(padData, chunkCount, expectedRoot, 0);
expect(merkleize(roots, chunkCount)).to.be.deep.equal(expectedRoot);
});
}
Expand Down

0 comments on commit 025a069

Please sign in to comment.