diff --git a/packages/client/bin/cli.ts b/packages/client/bin/cli.ts index fcffcbc80d6..ac9a99cb1c8 100755 --- a/packages/client/bin/cli.ts +++ b/packages/client/bin/cli.ts @@ -319,13 +319,8 @@ const args: ClientOpts = yargs(hideBin(process.argv)) boolean: true, default: true, }) - .option('disableBeaconSync', { - describe: - 'Disables beacon (optimistic) sync if the CL provides blocks at the head of the chain', - boolean: true, - }) - .option('forceSnapSync', { - describe: 'Force a snap sync run (for testing and development purposes)', + .option('enableSnapSync', { + describe: 'Enable snap state sync (for testing and development purposes)', boolean: true, }) .option('prefixStorageTrieKeys', { @@ -841,9 +836,8 @@ async function run() { port: args.port, saveReceipts: args.saveReceipts, syncmode: args.sync, - disableBeaconSync: args.disableBeaconSync, - forceSnapSync: args.forceSnapSync, prefixStorageTrieKeys: args.prefixStorageTrieKeys, + enableSnapSync: args.enableSnapSync, transports: args.transports, txLookupLimit: args.txLookupLimit, }) diff --git a/packages/client/src/config.ts b/packages/client/src/config.ts index f4acedf4f84..1e09fb6ae7f 100644 --- a/packages/client/src/config.ts +++ b/packages/client/src/config.ts @@ -43,22 +43,11 @@ export interface ConfigOptions { syncmode?: SyncMode /** - * Whether to disable beacon (optimistic) sync if CL provides - * blocks at the head of chain. + * Whether to enable and run snapSync, currently experimental * * Default: false */ - disableBeaconSync?: boolean - - /** - * Whether to test and run snapSync. When fully ready, this needs to - * be replaced by a more sophisticated condition based on how far back we are - * from the head, and how to run it in conjunction with the beacon sync - * blocks at the head of chain. - * - * Default: false - */ - forceSnapSync?: boolean + enableSnapSync?: boolean /** * A temporary option to offer backward compatibility with already-synced databases that are @@ -322,6 +311,8 @@ export interface ConfigOptions { engineNewpayloadMaxExecute?: number maxStorageRange?: bigint + + snapAvailabilityDepth?: bigint } export class Config { @@ -361,6 +352,7 @@ export class Config { // engine new payload calls can come in batch of 64, keeping 128 as the lookup factor public static readonly ENGINE_PARENTLOOKUP_MAX_DEPTH = 128 public static readonly ENGINE_NEWPAYLOAD_MAX_EXECUTE = 2 + public static readonly SNAP_AVAILABILITY_DEPTH = BigInt(128) public readonly logger: Logger public readonly syncmode: SyncMode @@ -404,18 +396,20 @@ export class Config { public readonly syncedStateRemovalPeriod: number public readonly engineParentLookupMaxDepth: number public readonly engineNewpayloadMaxExecute: number + public readonly snapAvailabilityDepth: bigint - public readonly disableBeaconSync: boolean - public readonly forceSnapSync: boolean - // Just a development only flag, will/should be removed - public readonly disableSnapSync: boolean = false public readonly prefixStorageTrieKeys: boolean + // Defaulting to false as experimental as of now + public readonly enableSnapSync: boolean public synchronized: boolean /** lastSyncDate in ms */ public lastSyncDate: number /** Best known block height */ public syncTargetHeight?: bigint + /** for snapsync */ + public snapTargetHeight?: bigint + public snapTargetRoot?: Uint8Array /** Client is in the process of shutting down */ public shutdown: boolean = false @@ -478,10 +472,10 @@ export class Config { options.engineParentLookupMaxDepth ?? Config.ENGINE_PARENTLOOKUP_MAX_DEPTH this.engineNewpayloadMaxExecute = options.engineNewpayloadMaxExecute ?? Config.ENGINE_NEWPAYLOAD_MAX_EXECUTE + this.snapAvailabilityDepth = options.snapAvailabilityDepth ?? Config.SNAP_AVAILABILITY_DEPTH - this.disableBeaconSync = options.disableBeaconSync ?? false - this.forceSnapSync = options.forceSnapSync ?? false this.prefixStorageTrieKeys = options.prefixStorageTrieKeys ?? true + this.enableSnapSync = options.enableSnapSync ?? false // Start it off as synchronized if this is configured to mine or as single node this.synchronized = this.isSingleNode ?? this.mine diff --git a/packages/client/src/rpc/modules/engine.ts b/packages/client/src/rpc/modules/engine.ts index 4f963248ebe..71ba5bad448 100644 --- a/packages/client/src/rpc/modules/engine.ts +++ b/packages/client/src/rpc/modules/engine.ts @@ -936,7 +936,7 @@ export class Engine { // It is possible that newPayload didn't start beacon sync as the payload it was asked to // evaluate didn't require syncing beacon. This can happen if the EL<>CL starts and CL // starts from a bit behind like how lodestar does - if (!this.service.beaconSync && !this.config.disableBeaconSync) { + if (!this.service.beaconSync) { await this.service.switchToBeaconSync() } diff --git a/packages/client/src/service/fullethereumservice.ts b/packages/client/src/service/fullethereumservice.ts index 868763f13b1..28964be20d6 100644 --- a/packages/client/src/service/fullethereumservice.ts +++ b/packages/client/src/service/fullethereumservice.ts @@ -31,13 +31,17 @@ interface FullEthereumServiceOptions extends ServiceOptions { * @memberof module:service */ export class FullEthereumService extends Service { - public synchronizer?: BeaconSynchronizer | FullSynchronizer | SnapSynchronizer + /* synchronizer for syncing the chain */ + public synchronizer?: BeaconSynchronizer | FullSynchronizer public lightserv: boolean public miner: Miner | undefined - public execution: VMExecution public txPool: TxPool public skeleton?: Skeleton + // objects dealing with state + public snapsync?: SnapSynchronizer + public execution: VMExecution + /** * Create new ETH service */ @@ -48,63 +52,67 @@ export class FullEthereumService extends Service { this.config.logger.info('Full sync mode') + const { metaDB } = options + if (metaDB !== undefined) { + this.skeleton = new Skeleton({ + config: this.config, + chain: this.chain, + metaDB, + }) + } + this.execution = new VMExecution({ config: options.config, stateDB: options.stateDB, - metaDB: options.metaDB, + metaDB, chain: this.chain, }) + this.snapsync = this.config.enableSnapSync + ? new SnapSynchronizer({ + config: this.config, + pool: this.pool, + chain: this.chain, + interval: this.interval, + skeleton: this.skeleton, + execution: this.execution, + }) + : undefined + this.txPool = new TxPool({ config: this.config, service: this, }) - const metaDB = (this.execution as any).metaDB - if (metaDB !== undefined) { - this.skeleton = new Skeleton({ - config: this.config, - chain: this.chain, - metaDB, - }) - } - - // This flag is just to run and test snap sync, when fully ready, this needs to - // be replaced by a more sophisticated condition based on how far back we are - // from the head, and how to run it in conjunction with the beacon sync - if (this.config.forceSnapSync) { - this.synchronizer = new SnapSynchronizer({ - config: this.config, - pool: this.pool, - chain: this.chain, - interval: this.interval, - }) - } else { + if (this.config.syncmode === SyncMode.Full) { if (this.config.chainCommon.gteHardfork(Hardfork.Paris) === true) { - if (!this.config.disableBeaconSync) { - void this.switchToBeaconSync() - } + this.synchronizer = new BeaconSynchronizer({ + config: this.config, + pool: this.pool, + chain: this.chain, + interval: this.interval, + execution: this.execution, + skeleton: this.skeleton!, + }) this.config.logger.info(`Post-merge 🐼 client mode: run with CL client.`) } else { - if (this.config.syncmode === SyncMode.Full) { - this.synchronizer = new FullSynchronizer({ + this.synchronizer = new FullSynchronizer({ + config: this.config, + pool: this.pool, + chain: this.chain, + txPool: this.txPool, + execution: this.execution, + interval: this.interval, + }) + + if (this.config.mine) { + this.miner = new Miner({ config: this.config, - pool: this.pool, - chain: this.chain, - txPool: this.txPool, - execution: this.execution, - interval: this.interval, + service: this, }) } } } - - if (this.config.mine) { - this.miner = new Miner({ - config: this.config, - service: this, - }) - } } /** @@ -171,7 +179,15 @@ export class FullEthereumService extends Service { if (txs[0].length > 0) this.txPool.sendNewTxHashes(txs, [peer]) }) await super.open() - await this.execution.open() + + // open snapsync instead of execution if instantiated + // it will open execution when done (or if doesn't need to snap sync) + if (this.snapsync !== undefined) { + await this.snapsync.open() + } else { + await this.execution.open() + } + this.txPool.open() if (this.config.mine) { // Start the TxPool immediately if mining @@ -203,7 +219,11 @@ export class FullEthereumService extends Service { this.txPool.stop() this.miner?.stop() await this.synchronizer?.stop() + + await this.snapsync?.stop() + // independently close execution even if it might have been opened by snapsync await this.execution.stop() + await super.stop() return true } diff --git a/packages/client/src/service/skeleton.ts b/packages/client/src/service/skeleton.ts index 59c0fa3eeb3..3655bb5b2d3 100644 --- a/packages/client/src/service/skeleton.ts +++ b/packages/client/src/service/skeleton.ts @@ -396,6 +396,16 @@ export class Skeleton extends MetaDBManager { return this.status.progress.subchains[0] } + async headHash(): Promise { + const subchain = this.bounds() + if (subchain !== undefined) { + const headBlock = await this.getBlock(subchain.head) + if (headBlock) { + return headBlock.hash() + } + } + } + private async trySubChainsMerge(): Promise { let merged = false let edited = false diff --git a/packages/client/src/sync/fetcher/accountfetcher.ts b/packages/client/src/sync/fetcher/accountfetcher.ts index 28489ae6695..64227c1d3d4 100644 --- a/packages/client/src/sync/fetcher/accountfetcher.ts +++ b/packages/client/src/sync/fetcher/accountfetcher.ts @@ -30,6 +30,7 @@ import type { EventBusType } from '../../types' import type { FetcherOptions } from './fetcher' import type { StorageRequest } from './storagefetcher' import type { Job } from './types' +import type { DefaultStateManager } from '@ethereumjs/statemanager' import type { Debugger } from 'debug' const { debug: createDebugLogger } = debugDefault @@ -51,6 +52,8 @@ export interface AccountFetcherOptions extends FetcherOptions { /** Destroy fetcher once all tasks are done */ destroyWhenDone?: boolean + + stateManager: DefaultStateManager } // root comes from block? @@ -112,6 +115,7 @@ export function snapFetchersCompleted( export class AccountFetcher extends Fetcher { protected debug: Debugger + stateManager: DefaultStateManager /** * The stateRoot for the fetcher which sorts of pin it to a snapshot. @@ -155,9 +159,12 @@ export class AccountFetcher extends Fetcher this.root = options.root this.first = options.first this.count = options.count ?? BIGINT_2 ** BIGINT_256 - this.first + this.stateManager = options.stateManager + this.codeTrie = new Trie({ useKeyHashing: true }) this.accountTrie = new Trie({ useKeyHashing: true }) this.accountToStorageTrie = new Map() + this.debug = createDebugLogger('client:AccountFetcher') this.storageFetcher = new StorageFetcher({ config: this.config, @@ -481,6 +488,10 @@ export class AccountFetcher extends Fetcher return tasks } + updateStateRoot(stateRoot: Uint8Array) { + this.root = stateRoot + } + nextTasks(): void { if ( this.in.length === 0 && diff --git a/packages/client/src/sync/snapsync.ts b/packages/client/src/sync/snapsync.ts index fb50a068d70..4a062c32b1d 100644 --- a/packages/client/src/sync/snapsync.ts +++ b/packages/client/src/sync/snapsync.ts @@ -1,25 +1,35 @@ -import { DefaultStateManager } from '@ethereumjs/statemanager' import { BIGINT_0, bytesToHex } from '@ethereumjs/util' import { Event } from '../types' +import { short } from '../util' import { AccountFetcher } from './fetcher' import { Synchronizer } from './sync' +import type { VMExecution } from '../execution' import type { Peer } from '../net/peer/peer' +import type { Skeleton } from '../service/skeleton' import type { SynchronizerOptions } from './sync' +import type { DefaultStateManager } from '@ethereumjs/statemanager' -interface SnapSynchronizerOptions extends SynchronizerOptions {} +interface SnapSynchronizerOptions extends SynchronizerOptions { + /** Skeleton chain */ + skeleton?: Skeleton + + /** VM Execution */ + execution: VMExecution +} export class SnapSynchronizer extends Synchronizer { public running = false - - stateManager: DefaultStateManager + skeleton?: Skeleton + private execution: VMExecution constructor(options: SnapSynchronizerOptions) { super(options) - this.stateManager = new DefaultStateManager() + this.skeleton = options.skeleton + this.execution = options.execution } /** @@ -82,8 +92,10 @@ export class SnapSynchronizer extends Synchronizer { * Get latest header of peer */ async latest(peer: Peer) { + // TODO: refine the way to query latest to fetch for the peer + const blockHash = this.skeleton?.headHash() ?? peer.eth!.status.bestHash const result = await peer.eth?.getBlockHeaders({ - block: peer.eth!.status.bestHash, + block: blockHash, max: 1, }) return result ? result[1][0] : undefined @@ -116,8 +128,18 @@ export class SnapSynchronizer extends Synchronizer { * @returns a boolean if the setup was successful */ async syncWithPeer(peer?: Peer): Promise { + // if skeleton is passed we have to wait for skeleton to be updated + if ( + this.skeleton !== undefined && + (!this.skeleton.isStarted() || this.skeleton.bounds() === undefined) + ) { + return false + } + const latest = peer ? await this.latest(peer) : undefined - if (!latest) return false + if (!latest) { + return false + } const stateRoot = latest.stateRoot const height = latest.number @@ -127,14 +149,35 @@ export class SnapSynchronizer extends Synchronizer { this.config.logger.info(`New sync target height=${height} hash=${bytesToHex(latest.hash())}`) } - this.fetcher = new AccountFetcher({ - config: this.config, - pool: this.pool, - root: stateRoot, - // This needs to be determined from the current state of the MPT dump - first: BIGINT_0, - }) + if (this.config.syncTargetHeight <= latest.number + this.config.snapAvailabilityDepth) { + if ((this.config.snapTargetHeight ?? BIGINT_0) < latest.number) { + this.config.snapTargetHeight = latest.number + this.config.snapTargetRoot = latest.stateRoot + } + if (this.fetcher === null || this.fetcher.syncErrored !== undefined) { + this.config.logger.info( + `syncWithPeer new AccountFetcher peer=${peer?.id} snapTargetHeight=${ + this.config.snapTargetHeight + } snapTargetRoot=${short(this.config.snapTargetRoot!)} ${ + this.fetcher === null + ? '' + : 'previous fetcher errored=' + this.fetcher.syncErrored?.message + }` + ) + this.fetcher = new AccountFetcher({ + config: this.config, + pool: this.pool, + stateManager: this.execution.vm.stateManager as DefaultStateManager, + root: stateRoot, + // This needs to be determined from the current state of the MPT dump + first: BigInt(0), + }) + } else { + this.config.logger.info(`syncWithPeer updating stateRoot=${short(stateRoot)}`) + this.fetcher.updateStateRoot(stateRoot) + } + } return true } diff --git a/packages/client/src/types.ts b/packages/client/src/types.ts index 9584ffec24e..b207faefb0b 100644 --- a/packages/client/src/types.ts +++ b/packages/client/src/types.ts @@ -152,9 +152,8 @@ export interface ClientOpts { dev?: boolean | string minerCoinbase?: Address saveReceipts?: boolean - disableBeaconSync?: boolean - forceSnapSync?: boolean prefixStorageTrieKeys?: boolean + enableSnapSync?: boolean txLookupLimit?: number startBlock?: number isSingleNode?: boolean diff --git a/packages/client/test/cli/cli.spec.ts b/packages/client/test/cli/cli.spec.ts index fa46bbfe064..28e0777a6b6 100644 --- a/packages/client/test/cli/cli.spec.ts +++ b/packages/client/test/cli/cli.spec.ts @@ -426,7 +426,7 @@ describe('[CLI]', () => { }, 30000) // test experimental feature options it('should start client when passed options for experimental features', async () => { - const cliArgs = ['--mine=true', '--forceSnapSync=true', '--dev=poa', '--port=30393'] + const cliArgs = ['--mine=true', '--enableSnapSync=true', '--dev=poa', '--port=30393'] const onData = async ( message: string, child: ChildProcessWithoutNullStreams, @@ -550,7 +550,6 @@ describe('[CLI]', () => { '--port=30301', '--dev=poa', '--isSingleNode=true', - '--disableBeaconSync=true', '--sync="none"', '--lightServe=true', '--mergeForkIdPostMerge=false', diff --git a/packages/client/test/service/fullethereumservice.spec.ts b/packages/client/test/service/fullethereumservice.spec.ts index d37d2155d09..0f8fe78d57f 100644 --- a/packages/client/test/service/fullethereumservice.spec.ts +++ b/packages/client/test/service/fullethereumservice.spec.ts @@ -370,7 +370,7 @@ describe('[FullEthereumService]', async () => { const chain = await Chain.create({ config }) let service = new FullEthereumService({ config, chain }) assert.ok(service.beaconSync, 'beacon sync should be available') - const configDisableBeaconSync = new Config({ transports: [], common, disableBeaconSync: true }) + const configDisableBeaconSync = new Config({ transports: [], common, syncmode: 'none' }) service = new FullEthereumService({ config: configDisableBeaconSync, chain }) assert.notOk(service.beaconSync, 'beacon sync should not be available') }) diff --git a/packages/client/test/sim/snapsync.spec.ts b/packages/client/test/sim/snapsync.spec.ts index e8683c815cd..54cef53afe7 100644 --- a/packages/client/test/sim/snapsync.spec.ts +++ b/packages/client/test/sim/snapsync.spec.ts @@ -259,7 +259,7 @@ async function createSnapClient( discDns: false, discV4: false, port: 30304, - forceSnapSync: true, + enableSnapSync: true, // Keep the single job sync range high as the state is not big maxAccountRange: (BigInt(2) ** BigInt(256) - BigInt(1)) / BigInt(10), maxFetcherJobs: 10,