From 1273b538628e2a526fe736818c271f1fe416c6af Mon Sep 17 00:00:00 2001 From: harkamal Date: Sun, 29 Sep 2024 22:54:45 +0530 Subject: [PATCH 01/15] client: add getBlobsV1 to the client to support CL blob import --- .../client/src/rpc/modules/engine/engine.ts | 17 ++++++++++ .../client/src/rpc/modules/engine/types.ts | 8 +++-- packages/client/src/service/txpool.ts | 31 +++++++++++++++++++ 3 files changed, 54 insertions(+), 2 deletions(-) diff --git a/packages/client/src/rpc/modules/engine/engine.ts b/packages/client/src/rpc/modules/engine/engine.ts index c5e6387ab8..f82a57cdd7 100644 --- a/packages/client/src/rpc/modules/engine/engine.ts +++ b/packages/client/src/rpc/modules/engine/engine.ts @@ -52,6 +52,7 @@ import type { Config } from '../../../config.js' import type { VMExecution } from '../../../execution/index.js' import type { FullEthereumService, Skeleton } from '../../../service/index.js' import type { + BlobAndProofV1, Bytes32, Bytes8, ExecutionPayloadBodyV1, @@ -316,6 +317,13 @@ export class Engine { ]), () => this.connectionManager.updateStatus(), ) + + this.getBlobsV1 = cmMiddleware( + middleware(callWithStackTrace(this.getBlobsV1.bind(this), this._rpcDebug), 1, [ + [validators.array(validators.bytes32)], + ]), + () => this.connectionManager.updateStatus(), + ) } /** @@ -1513,4 +1521,13 @@ export class Engine { } return payloads } + + private async getBlobsV1(params: [Bytes32]): Promise<(BlobAndProofV1 | null)[]> { + const blobsAndProof: (BlobAndProofV1 | null)[] = [] + for (const versionedHashHex of params) { + blobsAndProof.push(this.service.txPool.blobsAndProofsByHash.get(versionedHashHex) ?? null) + } + + return blobsAndProof + } } diff --git a/packages/client/src/rpc/modules/engine/types.ts b/packages/client/src/rpc/modules/engine/types.ts index bfab9b9750..811633b26d 100644 --- a/packages/client/src/rpc/modules/engine/types.ts +++ b/packages/client/src/rpc/modules/engine/types.ts @@ -20,8 +20,7 @@ export enum Status { export type Bytes8 = PrefixedHexString export type Bytes20 = PrefixedHexString export type Bytes32 = PrefixedHexString -// type Root = Bytes32 -export type Blob = Bytes32 +export type Blob = PrefixedHexString export type Bytes48 = PrefixedHexString export type Uint64 = PrefixedHexString export type Uint256 = PrefixedHexString @@ -81,6 +80,11 @@ export type ExecutionPayloadBodyV1 = { withdrawals: WithdrawalV1[] | null } +export type BlobAndProofV1 = { + blob: PrefixedHexString + proof: PrefixedHexString +} + export type ChainCache = { remoteBlocks: Map executedBlocks: Map diff --git a/packages/client/src/service/txpool.ts b/packages/client/src/service/txpool.ts index ec9dba5412..9378ec3be9 100644 --- a/packages/client/src/service/txpool.ts +++ b/packages/client/src/service/txpool.ts @@ -26,6 +26,7 @@ import type { PeerPool } from '../net/peerpool.js' import type { FullEthereumService } from './fullethereumservice.js' import type { Block } from '@ethereumjs/block' import type { FeeMarket1559Tx, LegacyTx, TypedTransaction } from '@ethereumjs/tx' +import type { PrefixedHexString } from '@ethereumjs/util' import type { VM } from '@ethereumjs/vm' // Configuration constants @@ -34,6 +35,8 @@ const MIN_GAS_PRICE = BigInt(100000000) // .1 GWei const TX_MAX_DATA_SIZE = 128 * 1024 // 128KB const MAX_POOL_SIZE = 5000 const MAX_TXS_PER_ACCOUNT = 100 +// keep for 1 epoch deep to handle reorgs +const BLOBS_AND_PROOFS_CACHE_LENGTH = 6 * 32 * 1 export interface TxPoolOptions { /* Config */ @@ -102,6 +105,10 @@ export class TxPool { * Maps an address to a `TxPoolObject` */ public pool: Map + public blobsAndProofsByHash: Map< + PrefixedHexString, + { blob: PrefixedHexString; proof: PrefixedHexString } + > /** * The number of txs currently in the pool @@ -167,6 +174,10 @@ export class TxPool { this.service = options.service this.pool = new Map() + this.blobsAndProofsByHash = new Map< + PrefixedHexString, + { blob: PrefixedHexString; proof: PrefixedHexString } + >() this.txsInPool = 0 this.handled = new Map() this.knownByPeer = new Map() @@ -371,6 +382,14 @@ export class TxPool { this.config.metrics?.feeMarketEIP1559TxGauge?.inc() } if (isBlob4844Tx(tx)) { + // add to blobs and proofs cache + for (const [i, versionedHash] of tx.blobVersionedHashes.entries()) { + const blob = tx.blobs![i] + const proof = tx.kzgProofs![i] + this.blobsAndProofsByHash.set(versionedHash, { blob, proof }) + } + this.pruneBlobsAndProofsCache() + this.config.metrics?.blobEIP4844TxGauge?.inc() } } catch (e) { @@ -379,6 +398,18 @@ export class TxPool { } } + pruneBlobsAndProofsCache() { + const pruneLength = this.blobsAndProofsByHash.size - BLOBS_AND_PROOFS_CACHE_LENGTH + let pruned = 0 + for (const versionedHash of this.blobsAndProofsByHash.keys()) { + if (pruned >= pruneLength) { + break + } + this.blobsAndProofsByHash.delete(versionedHash) + pruned++ + } + } + /** * Returns the available txs from the pool * @param txHashes From bc939aa9fe25eaf90c7c2f623cb16fdf337f7126 Mon Sep 17 00:00:00 2001 From: harkamal Date: Tue, 1 Oct 2024 21:20:54 +0530 Subject: [PATCH 02/15] add the getBlobV1 spec and debug/fix the api --- packages/client/src/rpc/modules/engine/engine.ts | 4 ++-- packages/client/test/rpc/engine/getPayloadV3.spec.ts | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/client/src/rpc/modules/engine/engine.ts b/packages/client/src/rpc/modules/engine/engine.ts index f82a57cdd7..571f01c9a2 100644 --- a/packages/client/src/rpc/modules/engine/engine.ts +++ b/packages/client/src/rpc/modules/engine/engine.ts @@ -1522,9 +1522,9 @@ export class Engine { return payloads } - private async getBlobsV1(params: [Bytes32]): Promise<(BlobAndProofV1 | null)[]> { + private async getBlobsV1(params: [[Bytes32]]): Promise<(BlobAndProofV1 | null)[]> { const blobsAndProof: (BlobAndProofV1 | null)[] = [] - for (const versionedHashHex of params) { + for (const versionedHashHex of params[0]) { blobsAndProof.push(this.service.txPool.blobsAndProofsByHash.get(versionedHashHex) ?? null) } diff --git a/packages/client/test/rpc/engine/getPayloadV3.spec.ts b/packages/client/test/rpc/engine/getPayloadV3.spec.ts index f91140b76c..c185478ea4 100644 --- a/packages/client/test/rpc/engine/getPayloadV3.spec.ts +++ b/packages/client/test/rpc/engine/getPayloadV3.spec.ts @@ -110,6 +110,16 @@ describe(method, () => { ).sign(pkey) await service.txPool.add(tx, true) + + // check the blob and proof is available via getBlobsV1 + res = await rpc.request('engine_getBlobsV1', [txVersionedHashes]) + const blobsAndProofs = res.result + for (let i = 0; i < txVersionedHashes.length; i++) { + const { blob, proof } = blobsAndProofs[i] + assert.equal(blob, txBlobs[i]) + assert.equal(proof, txProofs[i]) + } + res = await rpc.request('engine_getPayloadV3', [payloadId]) const { executionPayload, blobsBundle } = res.result From c66cebb6f8e329ba321d2f487b87abf99e341369 Mon Sep 17 00:00:00 2001 From: harkamal Date: Tue, 1 Oct 2024 21:40:56 +0530 Subject: [PATCH 03/15] debug and fix pending block spec and add assertions there as well --- packages/client/src/service/txpool.ts | 12 +++-- .../client/test/miner/pendingBlock.spec.ts | 51 ++++++++++++++----- 2 files changed, 44 insertions(+), 19 deletions(-) diff --git a/packages/client/src/service/txpool.ts b/packages/client/src/service/txpool.ts index 9378ec3be9..bde535cf72 100644 --- a/packages/client/src/service/txpool.ts +++ b/packages/client/src/service/txpool.ts @@ -383,12 +383,14 @@ export class TxPool { } if (isBlob4844Tx(tx)) { // add to blobs and proofs cache - for (const [i, versionedHash] of tx.blobVersionedHashes.entries()) { - const blob = tx.blobs![i] - const proof = tx.kzgProofs![i] - this.blobsAndProofsByHash.set(versionedHash, { blob, proof }) + if (tx.blobs !== undefined && tx.kzgProofs !== undefined) { + for (const [i, versionedHash] of tx.blobVersionedHashes.entries()) { + const blob = tx.blobs![i] + const proof = tx.kzgProofs![i] + this.blobsAndProofsByHash.set(versionedHash, { blob, proof }) + } + this.pruneBlobsAndProofsCache() } - this.pruneBlobsAndProofsCache() this.config.metrics?.blobEIP4844TxGauge?.inc() } diff --git a/packages/client/test/miner/pendingBlock.spec.ts b/packages/client/test/miner/pendingBlock.spec.ts index 88b78aecbe..c22a1c6931 100644 --- a/packages/client/test/miner/pendingBlock.spec.ts +++ b/packages/client/test/miner/pendingBlock.spec.ts @@ -28,7 +28,9 @@ import { mockBlockchain } from '../rpc/mockBlockchain.js' import type { Blockchain } from '@ethereumjs/blockchain' import type { TypedTransaction } from '@ethereumjs/tx' +import type { PrefixedHexString } from '@ethereumjs/util' import type { VM } from '@ethereumjs/vm' + const kzg = new microEthKZG(trustedSetup) const A = { @@ -354,23 +356,28 @@ describe('[PendingBlock]', async () => { const { txPool } = setup() - const blobs = getBlobs('hello world') - const commitments = blobsToCommitments(kzg, blobs) - const blobVersionedHashes = commitmentsToVersionedHashes(commitments) - const proofs = blobsToProofs(kzg, blobs, commitments) - - // Create 3 txs with 2 blobs each so that only 2 of them can be included in a build + // Create 2 txs with 3 blobs each so that only 2 of them can be included in a build + let blobs: PrefixedHexString = [], + proofs: PrefixedHexString[] = [], + versionedHashes: PrefixedHexString[] = [] for (let x = 0; x <= 2; x++) { + // generate unique blobs + const txBlobs = [ + ...getBlobs(`hello world-${x}1`), + ...getBlobs(`hello world-${x}2`), + ...getBlobs(`hello world-${x}3`), + ] + assert.equal(txBlobs.length, 3, '3 blobs should be created') + const txCommitments = blobsToCommitments(kzg, txBlobs) + const txBlobVersionedHashes = commitmentsToVersionedHashes(txCommitments) + const txProofs = blobsToProofs(kzg, txBlobs, txCommitments) + const txA01 = createBlob4844Tx( { - blobVersionedHashes: [ - ...blobVersionedHashes, - ...blobVersionedHashes, - ...blobVersionedHashes, - ], - blobs: [...blobs, ...blobs, ...blobs], - kzgCommitments: [...commitments, ...commitments, ...commitments], - kzgProofs: [...proofs, ...proofs, ...proofs], + blobVersionedHashes: txBlobVersionedHashes, + blobs: txBlobs, + kzgCommitments: txCommitments, + kzgProofs: txProofs, maxFeePerBlobGas: 100000000n, gasLimit: 0xffffffn, maxFeePerGas: 1000000000n, @@ -381,6 +388,22 @@ describe('[PendingBlock]', async () => { { common }, ).sign(A.privateKey) await txPool.add(txA01) + + // accumulate for verification + blobs = [...blobs, ...txBlobs] + proofs = [...proofs, ...txProofs] + versionedHashes = [...versionedHashes, ...txBlobVersionedHashes] + } + + // check if blobs and proofs are added in txpool by versioned hashes + for (let i = 0; i < versionedHashes.length; i++) { + const versionedHash = versionedHashes[i] + const blob = blobs[i] + const proof = proofs[i] + + const blobAndProof = txPool.blobsAndProofsByHash.get(versionedHash) ?? {} + assert.equal(blob, blobAndProof.blob, 'blob should match') + assert.equal(proof, blobAndProof.proof, 'proof should match') } // Add one other normal tx for nonce 3 which should also be not included in the build From 95e2c71926c82bacffa308e3aad0772a54f93f2b Mon Sep 17 00:00:00 2001 From: harkamal Date: Tue, 1 Oct 2024 22:08:08 +0530 Subject: [PATCH 04/15] apply feedback --- packages/client/src/config.ts | 16 ++++++++++++++++ packages/client/src/service/txpool.ts | 5 ++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/packages/client/src/config.ts b/packages/client/src/config.ts index 389e615e07..128fcc0f4a 100644 --- a/packages/client/src/config.ts +++ b/packages/client/src/config.ts @@ -339,6 +339,11 @@ export interface ConfigOptions { startExecution?: boolean ignoreStatelessInvalidExecs?: boolean + /** + * The cache for blobs and proofs to support CL import blocks + */ + blobsAndProofsCacheLength?: number + /** * Enables Prometheus Metrics that can be collected for monitoring client health */ @@ -393,6 +398,9 @@ export class Config { // randomly kept it at 5 for fast testing purposes but ideally should be >=32 slots public static readonly SNAP_TRANSITION_SAFE_DEPTH = BigInt(5) + // support blobs and proofs cache for CL getBlobs for upto 1 epoch of data + public static readonly BLOBS_AND_PROOFS_CACHE_BLOCKS = 32 + public readonly logger: Logger public readonly syncmode: SyncMode public readonly vm?: VM @@ -451,6 +459,8 @@ export class Config { public readonly startExecution: boolean public readonly ignoreStatelessInvalidExecs: boolean + public readonly blobsAndProofsCacheLength: number + public synchronized: boolean public lastSynchronized?: boolean /** lastSyncDate in ms */ @@ -553,6 +563,12 @@ export class Config { this.chainCommon = common.copy() this.execCommon = common.copy() + const blobGasLimit = common.param('maxblobGasPerBlock') + const blobGasPerBlob = common.param('blobGasPerBlob') + const allowedBlobs = Number(blobGasLimit / blobGasPerBlob) + this.blobsAndProofsCacheLength = + options.blobsAndProofsCacheLength ?? allowedBlobs * Config.BLOBS_AND_PROOFS_CACHE_BLOCKS + this.discDns = this.getDnsDiscovery(options.discDns) this.discV4 = options.discV4 ?? true diff --git a/packages/client/src/service/txpool.ts b/packages/client/src/service/txpool.ts index bde535cf72..2e13c42f50 100644 --- a/packages/client/src/service/txpool.ts +++ b/packages/client/src/service/txpool.ts @@ -35,8 +35,6 @@ const MIN_GAS_PRICE = BigInt(100000000) // .1 GWei const TX_MAX_DATA_SIZE = 128 * 1024 // 128KB const MAX_POOL_SIZE = 5000 const MAX_TXS_PER_ACCOUNT = 100 -// keep for 1 epoch deep to handle reorgs -const BLOBS_AND_PROOFS_CACHE_LENGTH = 6 * 32 * 1 export interface TxPoolOptions { /* Config */ @@ -401,8 +399,9 @@ export class TxPool { } pruneBlobsAndProofsCache() { - const pruneLength = this.blobsAndProofsByHash.size - BLOBS_AND_PROOFS_CACHE_LENGTH + const pruneLength = this.blobsAndProofsByHash.size - this.config.blobsAndProofsCacheLength let pruned = 0 + // since keys() is sorted by insertion order this prunes the olddest data in cache for (const versionedHash of this.blobsAndProofsByHash.keys()) { if (pruned >= pruneLength) { break From 653e47e82564d73987a2654def241bd097e8704d Mon Sep 17 00:00:00 2001 From: harkamal Date: Tue, 1 Oct 2024 22:12:37 +0530 Subject: [PATCH 05/15] fix the lint issues --- packages/client/test/miner/pendingBlock.spec.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/client/test/miner/pendingBlock.spec.ts b/packages/client/test/miner/pendingBlock.spec.ts index c22a1c6931..19a76e3bec 100644 --- a/packages/client/test/miner/pendingBlock.spec.ts +++ b/packages/client/test/miner/pendingBlock.spec.ts @@ -357,7 +357,7 @@ describe('[PendingBlock]', async () => { const { txPool } = setup() // Create 2 txs with 3 blobs each so that only 2 of them can be included in a build - let blobs: PrefixedHexString = [], + let blobs: PrefixedHexString[] = [], proofs: PrefixedHexString[] = [], versionedHashes: PrefixedHexString[] = [] for (let x = 0; x <= 2; x++) { @@ -401,7 +401,10 @@ describe('[PendingBlock]', async () => { const blob = blobs[i] const proof = proofs[i] - const blobAndProof = txPool.blobsAndProofsByHash.get(versionedHash) ?? {} + const blobAndProof = txPool.blobsAndProofsByHash.get(versionedHash) ?? { + blob: '0x0', + proof: '0x0', + } assert.equal(blob, blobAndProof.blob, 'blob should match') assert.equal(proof, blobAndProof.proof, 'proof should match') } From 35d2e07fc39b19a29e87458e407e918d22cbfc1b Mon Sep 17 00:00:00 2001 From: harkamal Date: Tue, 1 Oct 2024 22:15:48 +0530 Subject: [PATCH 06/15] spell checl --- packages/client/src/service/txpool.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/src/service/txpool.ts b/packages/client/src/service/txpool.ts index 2e13c42f50..e6073c3313 100644 --- a/packages/client/src/service/txpool.ts +++ b/packages/client/src/service/txpool.ts @@ -401,7 +401,7 @@ export class TxPool { pruneBlobsAndProofsCache() { const pruneLength = this.blobsAndProofsByHash.size - this.config.blobsAndProofsCacheLength let pruned = 0 - // since keys() is sorted by insertion order this prunes the olddest data in cache + // since keys() is sorted by insertion order this prunes the oldest data in cache for (const versionedHash of this.blobsAndProofsByHash.keys()) { if (pruned >= pruneLength) { break From 929e7d552cd22c43d5e692422535585b36621a31 Mon Sep 17 00:00:00 2001 From: harkamal Date: Tue, 1 Oct 2024 23:02:23 +0530 Subject: [PATCH 07/15] refac the cache length param to fix build --- packages/client/src/config.ts | 11 ++++------- packages/client/src/service/txpool.ts | 7 ++++++- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/client/src/config.ts b/packages/client/src/config.ts index 128fcc0f4a..e4291b9aac 100644 --- a/packages/client/src/config.ts +++ b/packages/client/src/config.ts @@ -342,7 +342,7 @@ export interface ConfigOptions { /** * The cache for blobs and proofs to support CL import blocks */ - blobsAndProofsCacheLength?: number + blobsAndProofsCacheBlocks?: number /** * Enables Prometheus Metrics that can be collected for monitoring client health @@ -459,7 +459,7 @@ export class Config { public readonly startExecution: boolean public readonly ignoreStatelessInvalidExecs: boolean - public readonly blobsAndProofsCacheLength: number + public readonly blobsAndProofsCacheBlocks: number public synchronized: boolean public lastSynchronized?: boolean @@ -563,11 +563,8 @@ export class Config { this.chainCommon = common.copy() this.execCommon = common.copy() - const blobGasLimit = common.param('maxblobGasPerBlock') - const blobGasPerBlob = common.param('blobGasPerBlob') - const allowedBlobs = Number(blobGasLimit / blobGasPerBlob) - this.blobsAndProofsCacheLength = - options.blobsAndProofsCacheLength ?? allowedBlobs * Config.BLOBS_AND_PROOFS_CACHE_BLOCKS + this.blobsAndProofsCacheBlocks = + options.blobsAndProofsCacheBlocks ?? Config.BLOBS_AND_PROOFS_CACHE_BLOCKS this.discDns = this.getDnsDiscovery(options.discDns) this.discV4 = options.discV4 ?? true diff --git a/packages/client/src/service/txpool.ts b/packages/client/src/service/txpool.ts index e6073c3313..dfa67c3063 100644 --- a/packages/client/src/service/txpool.ts +++ b/packages/client/src/service/txpool.ts @@ -399,7 +399,12 @@ export class TxPool { } pruneBlobsAndProofsCache() { - const pruneLength = this.blobsAndProofsByHash.size - this.config.blobsAndProofsCacheLength + const blobGasLimit = this.config.chainCommon.param('maxblobGasPerBlock') + const blobGasPerBlob = this.config.chainCommon.param('blobGasPerBlob') + const allowedBlobs = Number(blobGasLimit / blobGasPerBlob) + + const pruneLength = + this.blobsAndProofsByHash.size - allowedBlobs * this.config.blobsAndProofsCacheBlocks let pruned = 0 // since keys() is sorted by insertion order this prunes the oldest data in cache for (const versionedHash of this.blobsAndProofsByHash.keys()) { From b1b5fbd36c0c07885c551897d5e08ebb7aea7877 Mon Sep 17 00:00:00 2001 From: harkamal Date: Tue, 1 Oct 2024 23:20:58 +0530 Subject: [PATCH 08/15] fix breasking tests --- packages/client/src/service/txpool.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/client/src/service/txpool.ts b/packages/client/src/service/txpool.ts index dfa67c3063..61ba6ef7a0 100644 --- a/packages/client/src/service/txpool.ts +++ b/packages/client/src/service/txpool.ts @@ -399,12 +399,14 @@ export class TxPool { } pruneBlobsAndProofsCache() { - const blobGasLimit = this.config.chainCommon.param('maxblobGasPerBlock') - const blobGasPerBlob = this.config.chainCommon.param('blobGasPerBlob') - const allowedBlobs = Number(blobGasLimit / blobGasPerBlob) - + // currently common.param errors when not achieved cancun hardfork to do the below calculation + // const blobGasLimit = this.config.chainCommon.param('maxblobGasPerBlock') + // const blobGasPerBlob = this.config.chainCommon.param('blobGasPerBlob') + // + // A better way could be figured out to get allowedBlobsPerBlock + const allowedBlobsPerBlock = 6 const pruneLength = - this.blobsAndProofsByHash.size - allowedBlobs * this.config.blobsAndProofsCacheBlocks + this.blobsAndProofsByHash.size - allowedBlobsPerBlock * this.config.blobsAndProofsCacheBlocks let pruned = 0 // since keys() is sorted by insertion order this prunes the oldest data in cache for (const versionedHash of this.blobsAndProofsByHash.keys()) { From d2a2dc5cb03a74e066771c6d457ebbb3fe18ae11 Mon Sep 17 00:00:00 2001 From: harkamal Date: Thu, 3 Oct 2024 22:00:12 +0530 Subject: [PATCH 09/15] improvs --- packages/client/src/rpc/modules/engine/engine.ts | 10 ++++++++++ packages/client/src/service/txpool.ts | 10 ++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/packages/client/src/rpc/modules/engine/engine.ts b/packages/client/src/rpc/modules/engine/engine.ts index 571f01c9a2..3f866bb4ce 100644 --- a/packages/client/src/rpc/modules/engine/engine.ts +++ b/packages/client/src/rpc/modules/engine/engine.ts @@ -1523,6 +1523,16 @@ export class Engine { } private async getBlobsV1(params: [[Bytes32]]): Promise<(BlobAndProofV1 | null)[]> { + const blobGasLimit = this.config.chainCommon.param('maxblobGasPerBlock') + const blobGasPerBlob = this.config.chainCommon.param('blobGasPerBlob') + const allowedBlobsPerBlock = blobGasLimit / blobGasPerBlob + if (params[0].length > allowedBlobsPerBlock) { + throw { + code: TOO_LARGE_REQUEST, + message: `More than allowedBlobsPerBlock=${allowedBlobsPerBlock} hashes queried`, + } + } + const blobsAndProof: (BlobAndProofV1 | null)[] = [] for (const versionedHashHex of params[0]) { blobsAndProof.push(this.service.txPool.blobsAndProofsByHash.get(versionedHashHex) ?? null) diff --git a/packages/client/src/service/txpool.ts b/packages/client/src/service/txpool.ts index 61ba6ef7a0..6a1ef4201a 100644 --- a/packages/client/src/service/txpool.ts +++ b/packages/client/src/service/txpool.ts @@ -399,12 +399,10 @@ export class TxPool { } pruneBlobsAndProofsCache() { - // currently common.param errors when not achieved cancun hardfork to do the below calculation - // const blobGasLimit = this.config.chainCommon.param('maxblobGasPerBlock') - // const blobGasPerBlob = this.config.chainCommon.param('blobGasPerBlob') - // - // A better way could be figured out to get allowedBlobsPerBlock - const allowedBlobsPerBlock = 6 + const blobGasLimit = this.config.chainCommon.param('maxblobGasPerBlock') + const blobGasPerBlob = this.config.chainCommon.param('blobGasPerBlob') + const allowedBlobsPerBlock = Number(blobGasLimit / blobGasPerBlob) + const pruneLength = this.blobsAndProofsByHash.size - allowedBlobsPerBlock * this.config.blobsAndProofsCacheBlocks let pruned = 0 From bc576a3ebdd0f0472bb0a26198cc7e61080ed2bb Mon Sep 17 00:00:00 2001 From: harkamal Date: Fri, 4 Oct 2024 00:28:21 +0530 Subject: [PATCH 10/15] apply params fix for 4844 custom common --- packages/client/test/rpc/eth/sendRawTransaction.spec.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/client/test/rpc/eth/sendRawTransaction.spec.ts b/packages/client/test/rpc/eth/sendRawTransaction.spec.ts index f490e792fd..54830846b1 100644 --- a/packages/client/test/rpc/eth/sendRawTransaction.spec.ts +++ b/packages/client/test/rpc/eth/sendRawTransaction.spec.ts @@ -1,4 +1,4 @@ -import { BlockHeader } from '@ethereumjs/block' +import { BlockHeader, paramsBlock } from '@ethereumjs/block' import { Common, Hardfork, Mainnet, createCommonFromGethGenesis } from '@ethereumjs/common' import { MerkleStateManager } from '@ethereumjs/statemanager' import { createBlob4844Tx, createFeeMarket1559TxFromRLP, createLegacyTx } from '@ethereumjs/tx' @@ -228,7 +228,9 @@ describe(method, () => { chain: 'customChain', hardfork: Hardfork.Cancun, customCrypto: { kzg }, + params: paramsBlock, }) + common.setHardfork(Hardfork.Cancun) const { rpc, client } = await baseSetup({ commonChain: common, From 40125a00e1650ecb210c719d46adab7570718480 Mon Sep 17 00:00:00 2001 From: harkamal Date: Fri, 4 Oct 2024 15:45:33 +0530 Subject: [PATCH 11/15] fix spec --- packages/client/test/miner/pendingBlock.spec.ts | 1 + packages/client/test/rpc/helpers.ts | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/client/test/miner/pendingBlock.spec.ts b/packages/client/test/miner/pendingBlock.spec.ts index 19a76e3bec..092d28fb38 100644 --- a/packages/client/test/miner/pendingBlock.spec.ts +++ b/packages/client/test/miner/pendingBlock.spec.ts @@ -355,6 +355,7 @@ describe('[PendingBlock]', async () => { }) const { txPool } = setup() + txPool["config"].chainCommon.setHardfork(Hardfork.Cancun) // Create 2 txs with 3 blobs each so that only 2 of them can be included in a build let blobs: PrefixedHexString[] = [], diff --git a/packages/client/test/rpc/helpers.ts b/packages/client/test/rpc/helpers.ts index 73b7ec081d..dda35b4b3d 100644 --- a/packages/client/test/rpc/helpers.ts +++ b/packages/client/test/rpc/helpers.ts @@ -1,4 +1,4 @@ -import { createBlockHeader } from '@ethereumjs/block' +import { createBlockHeader, paramsBlock } from '@ethereumjs/block' import { createBlockchain } from '@ethereumjs/blockchain' import { Common, @@ -235,6 +235,7 @@ export async function setupChain(genesisFile: any, chainName = 'dev', clientOpts const common = createCommonFromGethGenesis(genesisFile, { chain: chainName, customCrypto: clientOpts.customCrypto, + params: paramsBlock, }) common.setHardforkBy({ blockNumber: 0, From 309ca44c1b89bb98b5d164a222783aed4e723999 Mon Sep 17 00:00:00 2001 From: harkamal Date: Fri, 4 Oct 2024 15:49:52 +0530 Subject: [PATCH 12/15] 128 len check --- packages/client/src/rpc/modules/engine/engine.ts | 7 ++----- packages/client/test/miner/pendingBlock.spec.ts | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/client/src/rpc/modules/engine/engine.ts b/packages/client/src/rpc/modules/engine/engine.ts index 3f866bb4ce..edc3e69432 100644 --- a/packages/client/src/rpc/modules/engine/engine.ts +++ b/packages/client/src/rpc/modules/engine/engine.ts @@ -1523,13 +1523,10 @@ export class Engine { } private async getBlobsV1(params: [[Bytes32]]): Promise<(BlobAndProofV1 | null)[]> { - const blobGasLimit = this.config.chainCommon.param('maxblobGasPerBlock') - const blobGasPerBlob = this.config.chainCommon.param('blobGasPerBlob') - const allowedBlobsPerBlock = blobGasLimit / blobGasPerBlob - if (params[0].length > allowedBlobsPerBlock) { + if (params[0].length > 128) { throw { code: TOO_LARGE_REQUEST, - message: `More than allowedBlobsPerBlock=${allowedBlobsPerBlock} hashes queried`, + message: `More than 128 hashes queried`, } } diff --git a/packages/client/test/miner/pendingBlock.spec.ts b/packages/client/test/miner/pendingBlock.spec.ts index 092d28fb38..7af737ff6a 100644 --- a/packages/client/test/miner/pendingBlock.spec.ts +++ b/packages/client/test/miner/pendingBlock.spec.ts @@ -355,7 +355,7 @@ describe('[PendingBlock]', async () => { }) const { txPool } = setup() - txPool["config"].chainCommon.setHardfork(Hardfork.Cancun) + txPool['config'].chainCommon.setHardfork(Hardfork.Cancun) // Create 2 txs with 3 blobs each so that only 2 of them can be included in a build let blobs: PrefixedHexString[] = [], From a2dc5ca3bf22e3302eb79861e261c28086fb2ccd Mon Sep 17 00:00:00 2001 From: harkamal Date: Fri, 4 Oct 2024 16:07:45 +0530 Subject: [PATCH 13/15] verify cache pruning in the test spec --- .../client/test/miner/pendingBlock.spec.ts | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/packages/client/test/miner/pendingBlock.spec.ts b/packages/client/test/miner/pendingBlock.spec.ts index 7af737ff6a..031e90d770 100644 --- a/packages/client/test/miner/pendingBlock.spec.ts +++ b/packages/client/test/miner/pendingBlock.spec.ts @@ -357,12 +357,31 @@ describe('[PendingBlock]', async () => { const { txPool } = setup() txPool['config'].chainCommon.setHardfork(Hardfork.Cancun) + // fill up the blobsAndProofsByHash and proofs cache before adding a blob tx + // for cache pruning check + const fillupBlobs = getBlobs('hello world') + const fillupCommitments = blobsToCommitments(kzg, fillupBlobs) + const fillupBlobVersionedHashes = commitmentsToVersionedHashes(fillupCommitments) + const fillupProofs = blobsToProofs(kzg, fillupBlobs, fillupCommitments) + const fillupBlobAndProof = { blob: fillupBlobs[0], proof: fillupProofs[0] } + + const blobGasLimit = txPool['config'].chainCommon.param('maxblobGasPerBlock') + const blobGasPerBlob = txPool['config'].chainCommon.param('blobGasPerBlob') + const allowedBlobsPerBlock = Number(blobGasLimit / blobGasPerBlob) + const allowedLength = allowedBlobsPerBlock * txPool['config'].blobsAndProofsCacheBlocks + + for (let i = 0; i < allowedLength; i++) { + // this is space efficent as same object is inserted in dummpy positions + txPool.blobsAndProofsByHash.set(`${fillupBlobVersionedHashes}-${i}`, fillupBlobAndProof) + } + assert.equal(txPool.blobsAndProofsByHash.size, allowedLength, 'fill the cache to capacity') + // Create 2 txs with 3 blobs each so that only 2 of them can be included in a build let blobs: PrefixedHexString[] = [], proofs: PrefixedHexString[] = [], versionedHashes: PrefixedHexString[] = [] for (let x = 0; x <= 2; x++) { - // generate unique blobs + // generate unique blobs different from fillupBlobs const txBlobs = [ ...getBlobs(`hello world-${x}1`), ...getBlobs(`hello world-${x}2`), @@ -396,6 +415,11 @@ describe('[PendingBlock]', async () => { versionedHashes = [...versionedHashes, ...txBlobVersionedHashes] } + assert.equal( + txPool.blobsAndProofsByHash.size, + allowedLength, + 'cache should be prune and stay at same size', + ) // check if blobs and proofs are added in txpool by versioned hashes for (let i = 0; i < versionedHashes.length; i++) { const versionedHash = versionedHashes[i] From d5d2b34a486e6ff1b076a76e0aed0f44223039f6 Mon Sep 17 00:00:00 2001 From: harkamal Date: Fri, 4 Oct 2024 16:15:31 +0530 Subject: [PATCH 14/15] lint/spell --- packages/client/test/miner/pendingBlock.spec.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/client/test/miner/pendingBlock.spec.ts b/packages/client/test/miner/pendingBlock.spec.ts index 031e90d770..9b4a0095a1 100644 --- a/packages/client/test/miner/pendingBlock.spec.ts +++ b/packages/client/test/miner/pendingBlock.spec.ts @@ -359,11 +359,11 @@ describe('[PendingBlock]', async () => { // fill up the blobsAndProofsByHash and proofs cache before adding a blob tx // for cache pruning check - const fillupBlobs = getBlobs('hello world') - const fillupCommitments = blobsToCommitments(kzg, fillupBlobs) - const fillupBlobVersionedHashes = commitmentsToVersionedHashes(fillupCommitments) - const fillupProofs = blobsToProofs(kzg, fillupBlobs, fillupCommitments) - const fillupBlobAndProof = { blob: fillupBlobs[0], proof: fillupProofs[0] } + const fillBlobs = getBlobs('hello world') + const fillCommitments = blobsToCommitments(kzg, fillBlobs) + const fillBlobVersionedHashes = commitmentsToVersionedHashes(fillCommitments) + const fillProofs = blobsToProofs(kzg, fillBlobs, fillCommitments) + const fillBlobAndProof = { blob: fillBlobs[0], proof: fillProofs[0] } const blobGasLimit = txPool['config'].chainCommon.param('maxblobGasPerBlock') const blobGasPerBlob = txPool['config'].chainCommon.param('blobGasPerBlob') @@ -371,8 +371,8 @@ describe('[PendingBlock]', async () => { const allowedLength = allowedBlobsPerBlock * txPool['config'].blobsAndProofsCacheBlocks for (let i = 0; i < allowedLength; i++) { - // this is space efficent as same object is inserted in dummpy positions - txPool.blobsAndProofsByHash.set(`${fillupBlobVersionedHashes}-${i}`, fillupBlobAndProof) + // this is space efficient as same object is inserted in dummy positions + txPool.blobsAndProofsByHash.set(`${fillBlobVersionedHashes}-${i}`, fillBlobAndProof) } assert.equal(txPool.blobsAndProofsByHash.size, allowedLength, 'fill the cache to capacity') @@ -381,7 +381,7 @@ describe('[PendingBlock]', async () => { proofs: PrefixedHexString[] = [], versionedHashes: PrefixedHexString[] = [] for (let x = 0; x <= 2; x++) { - // generate unique blobs different from fillupBlobs + // generate unique blobs different from fillBlobs const txBlobs = [ ...getBlobs(`hello world-${x}1`), ...getBlobs(`hello world-${x}2`), From ade3bc6b4ef526176adc632b34a9df5c3d76308c Mon Sep 17 00:00:00 2001 From: harkamal Date: Fri, 4 Oct 2024 16:23:24 +0530 Subject: [PATCH 15/15] fix typecheck --- packages/client/test/miner/pendingBlock.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/client/test/miner/pendingBlock.spec.ts b/packages/client/test/miner/pendingBlock.spec.ts index 9b4a0095a1..70857ebff9 100644 --- a/packages/client/test/miner/pendingBlock.spec.ts +++ b/packages/client/test/miner/pendingBlock.spec.ts @@ -12,6 +12,7 @@ import { commitmentsToVersionedHashes, getBlobs, hexToBytes, + intToHex, randomBytes, } from '@ethereumjs/util' import { createVM } from '@ethereumjs/vm' @@ -361,7 +362,6 @@ describe('[PendingBlock]', async () => { // for cache pruning check const fillBlobs = getBlobs('hello world') const fillCommitments = blobsToCommitments(kzg, fillBlobs) - const fillBlobVersionedHashes = commitmentsToVersionedHashes(fillCommitments) const fillProofs = blobsToProofs(kzg, fillBlobs, fillCommitments) const fillBlobAndProof = { blob: fillBlobs[0], proof: fillProofs[0] } @@ -372,7 +372,7 @@ describe('[PendingBlock]', async () => { for (let i = 0; i < allowedLength; i++) { // this is space efficient as same object is inserted in dummy positions - txPool.blobsAndProofsByHash.set(`${fillBlobVersionedHashes}-${i}`, fillBlobAndProof) + txPool.blobsAndProofsByHash.set(intToHex(i), fillBlobAndProof) } assert.equal(txPool.blobsAndProofsByHash.size, allowedLength, 'fill the cache to capacity')