blockchain: allow optimistic block insertion in blockchain

packages/blockchain/src/blockchain.ts

@@ -25,6 +25,7 @@ import {
 import { DBManager } from './db/manager.js'
 import { DBTarget } from './db/operation.js'
 
+import type { OptimisticOpts } from './db/operation.js'
 import type {
  BlockchainEvents,
  BlockchainInterface,
@@ -272,8 +273,8 @@ export class Blockchain implements BlockchainInterface {
  * heads/hashes are overwritten.
  * @param block - The block to be added to the blockchain
  */
- async putBlock(block: Block) {
- await this._putBlockOrHeader(block)
+ async putBlock(block: Block, opts?: OptimisticOpts) {
+ await this._putBlockOrHeader(block, opts)
  }
 
  /**
@@ -344,7 +345,7 @@ export class Blockchain implements BlockchainInterface {
  * header using the iterator method.
  * @hidden
  */
- private async _putBlockOrHeader(item: Block | BlockHeader) {
+ private async _putBlockOrHeader(item: Block | BlockHeader, optimisticOpts?: OptimisticOpts) {
  await this.runWithLock<void>(async () => {
  // Save the current sane state incase _putBlockOrHeader midway with some
  // dirty changes in head trackers
@@ -362,96 +363,119 @@ export class Blockchain implements BlockchainInterface {
  if (isGenesis) {
  if (equalsBytes(this.genesisBlock.hash(), block.hash())) {
  // Try to re-put the existing genesis block, accept this
+ // genesis block is not optimistic
+ optimisticOpts = undefined
  return
  }
  throw new Error(
  'Cannot put a different genesis block than current blockchain genesis: create a new Blockchain',
  )
  }
 
- const { header } = block
- const blockHash = header.hash()
- const blockNumber = header.number
- let td = header.difficulty
- const currentTd = { header: BIGINT_0, block: BIGINT_0 }
- let dbOps: DBOp[] = []
-
  if (block.common.chainId() !== this.common.chainId()) {
  throw new Error(
  `Chain mismatch while trying to put block or header. Chain ID of block: ${block.common.chainId}, chain ID of blockchain : ${this.common.chainId}`,
  )
  }
 
- if (this._validateBlocks && !isGenesis && item instanceof Block) {
+ if (
+ this._validateBlocks &&
+ !isGenesis &&
+ item instanceof Block &&
+ optimisticOpts === undefined
+ ) {
  // this calls into `getBlock`, which is why we cannot lock yet
  await this.validateBlock(block)
  }
 
- if (this._validateConsensus) {
+ if (this._validateConsensus && optimisticOpts === undefined) {
  await this.consensus!.validateConsensus(block)
  }
 
- // set total difficulty in the current context scope
- if (this._headHeaderHash) {
- currentTd.header = await this.getTotalDifficulty(this._headHeaderHash)
- }
- if (this._headBlockHash) {
- currentTd.block = await this.getTotalDifficulty(this._headBlockHash)
- }
-
  // calculate the total difficulty of the new block
- const parentTd = await this.getParentTD(header)
- if (!block.isGenesis()) {
- td += parentTd
- }
-
- // save total difficulty to the database
- dbOps = dbOps.concat(DBSetTD(td, blockNumber, blockHash))
-
- // save header/block to the database, but save the input not our wrapper block
- dbOps = dbOps.concat(DBSetBlockOrHeader(item))
-
- let commonAncestor: undefined | BlockHeader
- let ancestorHeaders: undefined | BlockHeader[]
- // if total difficulty is higher than current, add it to canonical chain
- if (
- block.isGenesis() ||
- td > currentTd.header ||
- block.common.consensusType() === ConsensusType.ProofOfStake
- ) {
- const foundCommon = await this.findCommonAncestor(header)
- commonAncestor = foundCommon.commonAncestor
- ancestorHeaders = foundCommon.ancestorHeaders
+ const { header } = block
+ const blockHash = header.hash()
+ const blockNumber = header.number
 
- this._headHeaderHash = blockHash
- if (item instanceof Block) {
- this._headBlockHash = blockHash
+ let td = header.difficulty
+ try {
+ const parentTd = await this.getParentTD(header)
+ if (!block.isGenesis()) {
+ td += parentTd
  }
- if (this._hardforkByHeadBlockNumber) {
- await this.checkAndTransitionHardForkByNumber(blockNumber, header.timestamp)
+ } catch (e) {
+ // opimistic insertion does care about td
+ if (optimisticOpts === undefined) {
+ throw e
  }
+ }
 
- // delete higher number assignments and overwrite stale canonical chain
- await this._deleteCanonicalChainReferences(blockNumber + BIGINT_1, blockHash, dbOps)
- // from the current header block, check the blockchain in reverse (i.e.
- // traverse `parentHash`) until `numberToHash` matches the current
- // number/hash in the canonical chain also: overwrite any heads if these
- // heads are stale in `_heads` and `_headBlockHash`
- await this._rebuildCanonical(header, dbOps)
+ let dbOps: DBOp[] = []
+ if (optimisticOpts !== undefined) {
+ dbOps = dbOps.concat(DBSetBlockOrHeader(item))
+ dbOps.push(DBSetHashToNumber(blockHash, blockNumber))
+ if (optimisticOpts.fcUed) {
+ dbOps.push(DBOp.set(DBTarget.OptimisticNumberToHash, blockHash, { blockNumber }))
+ }
+ await this.dbManager.batch(dbOps)
  } else {
- // the TD is lower than the current highest TD so we will add the block
- // to the DB, but will not mark it as the canonical chain.
- if (td > currentTd.block && item instanceof Block) {
- this._headBlockHash = blockHash
+ const currentTd = { header: BIGINT_0, block: BIGINT_0 }
+ // set total difficulty in the current context scope
+ if (this._headHeaderHash) {
+ currentTd.header = await this.getTotalDifficulty(this._headHeaderHash)
+ }
+ if (this._headBlockHash) {
+ currentTd.block = await this.getTotalDifficulty(this._headBlockHash)
  }
- // save hash to number lookup info even if rebuild not needed
- dbOps.push(DBSetHashToNumber(blockHash, blockNumber))
- }
 
- const ops = dbOps.concat(this._saveHeadOps())
- await this.dbManager.batch(ops)
+ // save total difficulty to the database
+ dbOps = dbOps.concat(DBSetTD(td, blockNumber, blockHash))
+
+ // save header/block to the database, but save the input not our wrapper block
+ dbOps = dbOps.concat(DBSetBlockOrHeader(item))
+
+ let commonAncestor: undefined | BlockHeader
+ let ancestorHeaders: undefined | BlockHeader[]
+ // if total difficulty is higher than current, add it to canonical chain
+ if (
+ block.isGenesis() ||
+ td > currentTd.header ||
+ block.common.consensusType() === ConsensusType.ProofOfStake
+ ) {
+ const foundCommon = await this.findCommonAncestor(header)
+ commonAncestor = foundCommon.commonAncestor
+ ancestorHeaders = foundCommon.ancestorHeaders
+
+ this._headHeaderHash = blockHash
+ if (item instanceof Block) {
+ this._headBlockHash = blockHash
+ }
+ if (this._hardforkByHeadBlockNumber) {
+ await this.checkAndTransitionHardForkByNumber(blockNumber, header.timestamp)
+ }
 
- await this.consensus?.newBlock(block, commonAncestor, ancestorHeaders)
+ // delete higher number assignments and overwrite stale canonical chain
+ await this._deleteCanonicalChainReferences(blockNumber + BIGINT_1, blockHash, dbOps)
+ // from the current header block, check the blockchain in reverse (i.e.
+ // traverse `parentHash`) until `numberToHash` matches the current
+ // number/hash in the canonical chain also: overwrite any heads if these
+ // heads are stale in `_heads` and `_headBlockHash`
+ await this._rebuildCanonical(header, dbOps)
+ } else {
+ // the TD is lower than the current highest TD so we will add the block
+ // to the DB, but will not mark it as the canonical chain.
+ if (td > currentTd.block && item instanceof Block) {
+ this._headBlockHash = blockHash
+ }
+ // save hash to number lookup info even if rebuild not needed
+ dbOps.push(DBSetHashToNumber(blockHash, blockNumber))
+ }
+
+ const ops = dbOps.concat(this._saveHeadOps())
+ await this.dbManager.batch(ops)
+
+ await this.consensus?.newBlock(block, commonAncestor, ancestorHeaders)
+ }
  } catch (e) {
  // restore head to the previously sane state
  this._heads = oldHeads
@@ -658,13 +682,16 @@ export class Blockchain implements BlockchainInterface {
  * this will be immediately looked up, otherwise it will wait until we have
  * unlocked the DB
  */
- async getBlock(blockId: Uint8Array | number | bigint): Promise<Block> {
+ async getBlock(
+ blockId: Uint8Array | number | bigint,
+ optimisticOpts?: OptimisticOpts,
+ ): Promise<Block> {
  // cannot wait for a lock here: it is used both in `validate` of `Block`
  // (calls `getBlock` to get `parentHash`) it is also called from `runBlock`
  // in the `VM` if we encounter a `BLOCKHASH` opcode: then a bigint is used we
  // need to then read the block from the canonical chain Q: is this safe? We
  // know it is OK if we call it from the iterator... (runBlock)
- const block = await this.dbManager.getBlock(blockId)
+ const block = await this.dbManager.getBlock(blockId, optimisticOpts)
 
  if (block === undefined) {
  if (typeof blockId === 'object') {

packages/blockchain/src/db/constants.ts

@@ -28,6 +28,7 @@ const TD_SUFFIX = utf8ToBytes('t')
  * headerPrefix + number + numSuffix -> hash
  */
 const NUM_SUFFIX = utf8ToBytes('n')
+const OPTIMISTIC_NUM_SUFFIX = utf8ToBytes('o')
 
 /**
  * blockHashPrefix + hash -> number
@@ -55,6 +56,9 @@ const bodyKey = (n: bigint, hash: Uint8Array) => concatBytes(BODY_PREFIX, bytesB
 
 const numberToHashKey = (n: bigint) => concatBytes(HEADER_PREFIX, bytesBE8(n), NUM_SUFFIX)
 
+const optimisticNumberToHashKey = (n: bigint) =>
+ concatBytes(HEADER_PREFIX, bytesBE8(n), OPTIMISTIC_NUM_SUFFIX)
+
 const hashToNumberKey = (hash: Uint8Array) => concatBytes(BLOCK_HASH_PREFIX, hash)
 
 /**
@@ -69,5 +73,6 @@ export {
  headerKey,
  HEADS_KEY,
  numberToHashKey,
+ optimisticNumberToHashKey,
  tdKey,
 }

packages/blockchain/src/db/manager.ts

@@ -12,7 +12,7 @@ import {
 import { Cache } from './cache.js'
 import { DBOp, DBTarget } from './operation.js'
 
-import type { DatabaseKey } from './operation.js'
+import type { DatabaseKey, OptimisticOpts } from './operation.js'
 import type { Block, BlockBodyBytes, BlockBytes, BlockOptions } from '@ethereumjs/block'
 import type { Common } from '@ethereumjs/common'
 import type { BatchDBOp, DB, DBObject, DelBatch, PutBatch } from '@ethereumjs/util'
@@ -47,6 +47,7 @@ export class DBManager {
  body: new Cache({ max: 256 }),
  numberToHash: new Cache({ max: 2048 }),
  hashToNumber: new Cache({ max: 2048 }),
+ optimisticNumberToHash: new Cache({ max: 2048 }),
  }
  }
 
@@ -84,7 +85,10 @@ export class DBManager {
  * Fetches a block (header and body) given a block id,
  * which can be either its hash or its number.
  */
- async getBlock(blockId: Uint8Array | bigint | number): Promise<Block | undefined> {
+ async getBlock(
+ blockId: Uint8Array | bigint | number,
+ optimisticOpts?: OptimisticOpts,
+ ): Promise<Block | undefined> {
  if (typeof blockId === 'number' && Number.isInteger(blockId)) {
  blockId = BigInt(blockId)
  }
@@ -95,9 +99,32 @@ export class DBManager {
  if (blockId instanceof Uint8Array) {
  hash = blockId
  number = await this.hashToNumber(blockId)
+ if (number === undefined) {
+ return undefined
+ }
+
+ if (optimisticOpts?.fcUed === true) {
+ let optimisticHash = await this.optimisticNumberToHash(number)
+ if (optimisticHash === undefined && optimisticOpts.linked === true) {
+ optimisticHash = await this.numberToHash(number)
+ }
+ if (optimisticHash === undefined || !equalsBytes(optimisticHash, hash)) {
+ return undefined
+ }
+ }
  } else if (typeof blockId === 'bigint') {
  number = blockId
- hash = await this.numberToHash(blockId)
+ if (optimisticOpts !== undefined) {
+ if (!optimisticOpts.fcUed) {
+ throw Error(`Invalid fcUed optimistic block by number lookup`)
+ }
+ hash = await this.optimisticNumberToHash(blockId)
+ if (hash === undefined && optimisticOpts.linked === true) {
+ hash = await this.numberToHash(blockId)
+ }
+ } else {
+ hash = await this.numberToHash(blockId)
+ }
  } else {
  throw new Error('Unknown blockId type')
  }
@@ -190,6 +217,11 @@ export class DBManager {
  return value
  }
 
+ async optimisticNumberToHash(blockNumber: bigint): Promise<Uint8Array | undefined> {
+ const value = await this.get(DBTarget.OptimisticNumberToHash, { blockNumber })
+ return value
+ }
+
  /**
  * Fetches a key from the db. If `opts.cache` is specified
  * it first tries to load from cache, and on cache miss will

packages/blockchain/src/db/operation.ts

@@ -8,11 +8,14 @@ import {
  hashToNumberKey,
  headerKey,
  numberToHashKey,
+ optimisticNumberToHashKey,
  tdKey,
 } from './constants.js'
 
 import type { CacheMap } from './manager.js'
 
+export type OptimisticOpts = { fcUed: boolean; linked?: boolean }
+
 export enum DBTarget {
  Heads,
  HeadHeader,
@@ -25,6 +28,7 @@ export enum DBTarget {
  CliqueSignerStates,
  CliqueVotes,
  CliqueBlockSigners,
+ OptimisticNumberToHash,
 }
 
 /**
@@ -88,6 +92,11 @@ export class DBOp {
  this.cacheString = 'numberToHash'
  break
  }
+ case DBTarget.OptimisticNumberToHash: {
+ this.baseDBOp.key = optimisticNumberToHashKey(key!.blockNumber!)
+ this.cacheString = 'optimisticNumberToHash'
+ break
+ }
  case DBTarget.TotalDifficulty: {
  this.baseDBOp.key = tdKey(key!.blockNumber!, key!.blockHash!)
  this.cacheString = 'td'

packages/blockchain/src/types.ts

@@ -1,3 +1,4 @@
+import type { OptimisticOpts } from './db/operation.js'
 import type { Blockchain } from './index.js'
 import type { Block, BlockHeader } from '@ethereumjs/block'
 import type { Common, ConsensusAlgorithm } from '@ethereumjs/common'
@@ -16,7 +17,7 @@ export interface BlockchainInterface {
  *
  * @param block - The block to be added to the blockchain.
  */
- putBlock(block: Block): Promise<void>
+ putBlock(block: Block, optimisticOpts?: OptimisticOpts): Promise<void>
 
  /**
  * Deletes a block from the blockchain. All child blocks in the chain are
@@ -29,7 +30,7 @@ export interface BlockchainInterface {
  /**
  * Returns a block by its hash or number.
  */
- getBlock(blockId: Uint8Array | number | bigint): Promise<Block>
+ getBlock(blockId: Uint8Array | number | bigint, optimisticOpts?: OptimisticOpts): Promise<Block>
 
  /**
  * Iterates through blocks starting at the specified iterator head and calls