diff --git a/packages/blockchain/src/blockchain.ts b/packages/blockchain/src/blockchain.ts index 9b7280ad56..0e1ae42477 100644 --- a/packages/blockchain/src/blockchain.ts +++ b/packages/blockchain/src/blockchain.ts @@ -981,44 +981,61 @@ export class Blockchain implements BlockchainInterface { let blocksRanCounter = 0 let lastBlock: Block | undefined - while (maxBlocks !== blocksRanCounter) { - try { - let nextBlock = await this.getBlock(nextBlockNumber) - const reorg = lastBlock - ? !equalsBytes(lastBlock.hash(), nextBlock.header.parentHash) - : false - if (reorg) { - // If reorg has happened, the _heads must have been updated so lets reload the counters - headHash = this._heads[name] ?? this.genesisBlock.hash() - headBlockNumber = await this.dbManager.hashToNumber(headHash) - nextBlockNumber = headBlockNumber + BigInt(1) - nextBlock = await this.getBlock(nextBlockNumber) - } - this._heads[name] = nextBlock.hash() - lastBlock = nextBlock - if (releaseLockOnCallback === true) { - this._lock.release() - } + try { + while (maxBlocks !== blocksRanCounter) { try { - await onBlock(nextBlock, reorg) - } finally { + let nextBlock = await this.getBlock(nextBlockNumber) + const reorg = lastBlock + ? !equalsBytes(lastBlock.hash(), nextBlock.header.parentHash) + : false + if (reorg) { + // If reorg has happened, the _heads must have been updated so lets reload the counters + headHash = this._heads[name] ?? this.genesisBlock.hash() + headBlockNumber = await this.dbManager.hashToNumber(headHash) + nextBlockNumber = headBlockNumber + BigInt(1) + nextBlock = await this.getBlock(nextBlockNumber) + } + + // While running onBlock with released lock, reorgs can happen via putBlocks + let reorgWhileOnBlock = false if (releaseLockOnCallback === true) { - await this._lock.acquire() + this._lock.release() + } + try { + await onBlock(nextBlock, reorg) + } finally { + if (releaseLockOnCallback === true) { + await this._lock.acquire() + // If lock was released check if reorg occured + const nextBlockMayBeReorged = await this.getBlock(nextBlockNumber).catch( + (_e) => null + ) + reorgWhileOnBlock = nextBlockMayBeReorged + ? !equalsBytes(nextBlockMayBeReorged.hash(), nextBlock.hash()) + : true + } + } + + // if there was no reorg, update head + if (!reorgWhileOnBlock) { + this._heads[name] = nextBlock.hash() + lastBlock = nextBlock + nextBlockNumber++ + } + // Successful execution of onBlock, move the head pointer + blocksRanCounter++ + } catch (error: any) { + if (error.code === 'LEVEL_NOT_FOUND') { + break + } else { + throw error } - } - nextBlockNumber++ - blocksRanCounter++ - } catch (error: any) { - if (error.code === 'LEVEL_NOT_FOUND') { - break - } else { - throw error } } + return blocksRanCounter + } finally { + await this._saveHeads() } - - await this._saveHeads() - return blocksRanCounter }) } diff --git a/packages/client/lib/execution/vmexecution.ts b/packages/client/lib/execution/vmexecution.ts index 65aa39b1d7..c28b477324 100644 --- a/packages/client/lib/execution/vmexecution.ts +++ b/packages/client/lib/execution/vmexecution.ts @@ -34,7 +34,7 @@ export class VMExecution extends Execution { public receiptsManager?: ReceiptsManager private pendingReceipts?: Map - private vmPromise?: Promise + private vmPromise?: Promise /** Maximally tolerated block time before giving a warning on console */ private MAX_TOLERATED_BLOCK_TIME = 12 @@ -268,7 +268,7 @@ export class VMExecution extends Execution { async run(loop = true, runOnlybatched = false): Promise { if (this.running || !this.started || this.shutdown) return 0 this.running = true - let numExecuted: number | undefined + let numExecuted: number | null | undefined = undefined const { blockchain } = this.vm if (typeof blockchain.getIteratorHead !== 'function') { @@ -302,82 +302,105 @@ export class VMExecution extends Execution { headBlock = undefined parentState = undefined errorBlock = undefined - this.vmPromise = blockchain.iterator( - 'vm', - async (block: Block, reorg: boolean) => { - if (errorBlock !== undefined) return - // determine starting state for block run - // if we are just starting or if a chain reorg has happened - if (headBlock === undefined || reorg) { - const headBlock = await blockchain.getBlock(block.header.parentHash) - parentState = headBlock.header.stateRoot - - if (reorg) { - this.config.logger.info( - `Chain reorg happened, set new head to block number=${headBlock.header.number}, clearing state cache for VM execution.` - ) - } - } - // run block, update head if valid - try { - const { number, timestamp } = block.header - if (typeof blockchain.getTotalDifficulty !== 'function') { - throw new Error( - 'cannot get iterator head: blockchain has no getTotalDifficulty function' - ) + this.vmPromise = blockchain + .iterator( + 'vm', + async (block: Block, reorg: boolean) => { + // determine starting state for block run + // if we are just starting or if a chain reorg has happened + if (headBlock === undefined || reorg) { + const headBlock = await blockchain.getBlock(block.header.parentHash) + parentState = headBlock.header.stateRoot + + if (reorg) { + this.config.logger.info( + `Chain reorg happened, set new head to block number=${headBlock.header.number}, clearing state cache for VM execution.` + ) + } } - const td = await blockchain.getTotalDifficulty(block.header.parentHash) - const hardfork = this.config.execCommon.getHardforkByBlockNumber(number, td, timestamp) - if (hardfork !== this.hardfork) { - const hash = short(block.hash()) - this.config.logger.info( - `Execution hardfork switch on block number=${number} hash=${hash} old=${this.hardfork} new=${hardfork}` - ) - this.hardfork = this.config.execCommon.setHardforkByBlockNumber(number, td, timestamp) - } - let skipBlockValidation = false - if (this.config.execCommon.consensusType() === ConsensusType.ProofOfAuthority) { - // Block validation is redundant here and leads to consistency problems - // on PoA clique along blockchain-including validation checks - // (signer states might have moved on when sync is ahead) - skipBlockValidation = true - } + // run block, update head if valid + try { + const { number, timestamp } = block.header + if (typeof blockchain.getTotalDifficulty !== 'function') { + throw new Error( + 'cannot get iterator head: blockchain has no getTotalDifficulty function' + ) + } + const td = await blockchain.getTotalDifficulty(block.header.parentHash) - await this.runWithLock(async () => { - // we are skipping header validation because the block has been picked from the - // blockchain and header should have already been validated while putBlock - if (!this.started) { - throw Error('Execution stopped') + const hardfork = this.config.execCommon.getHardforkByBlockNumber( + number, + td, + timestamp + ) + if (hardfork !== this.hardfork) { + const hash = short(block.hash()) + this.config.logger.info( + `Execution hardfork switch on block number=${number} hash=${hash} old=${this.hardfork} new=${hardfork}` + ) + this.hardfork = this.config.execCommon.setHardforkByBlockNumber( + number, + td, + timestamp + ) } - const beforeTS = Date.now() - this.stats(this.vm) - const result = await this.vm.runBlock({ - block, - root: parentState, - clearCache: reorg ? true : false, - skipBlockValidation, - skipHeaderValidation: true, - }) - const afterTS = Date.now() - const diffSec = Math.round((afterTS - beforeTS) / 1000) - - if (diffSec > this.MAX_TOLERATED_BLOCK_TIME) { - const msg = `Slow block execution for block num=${ - block.header.number - } hash=0x${bytesToHex(block.hash())} txs=${block.transactions.length} gasUsed=${ - result.gasUsed - } time=${diffSec}secs` - this.config.logger.warn(msg) + let skipBlockValidation = false + if (this.config.execCommon.consensusType() === ConsensusType.ProofOfAuthority) { + // Block validation is redundant here and leads to consistency problems + // on PoA clique along blockchain-including validation checks + // (signer states might have moved on when sync is ahead) + skipBlockValidation = true } - void this.receiptsManager?.saveReceipts(block, result.receipts) - }) - txCounter += block.transactions.length - // set as new head block - headBlock = block - parentState = block.header.stateRoot - } catch (error: any) { + await this.runWithLock(async () => { + // we are skipping header validation because the block has been picked from the + // blockchain and header should have already been validated while putBlock + if (!this.started) { + throw Error('Execution stopped') + } + const beforeTS = Date.now() + this.stats(this.vm) + const result = await this.vm.runBlock({ + block, + root: parentState, + clearCache: reorg ? true : false, + skipBlockValidation, + skipHeaderValidation: true, + }) + const afterTS = Date.now() + const diffSec = Math.round((afterTS - beforeTS) / 1000) + + if (diffSec > this.MAX_TOLERATED_BLOCK_TIME) { + const msg = `Slow block execution for block num=${ + block.header.number + } hash=0x${bytesToHex(block.hash())} txs=${block.transactions.length} gasUsed=${ + result.gasUsed + } time=${diffSec}secs` + this.config.logger.warn(msg) + } + + void this.receiptsManager?.saveReceipts(block, result.receipts) + }) + + txCounter += block.transactions.length + // set as new head block + headBlock = block + parentState = block.header.stateRoot + } catch (error: any) { + // Store error block and throw which will make iterator stop, exit and save + // last successfully executed head as vmHead + errorBlock = block + throw error + } + }, + this.config.numBlocksPerIteration, + // release lock on this callback so other blockchain ops can happen while this block is being executed + true + ) + // Ensure to catch and not throw as this would lead to unCaughtException with process exit + .catch(async (error) => { + if (errorBlock !== undefined) { // TODO: determine if there is a way to differentiate between the cases // a) a bad block is served by a bad peer -> delete the block and restart sync // sync from parent block @@ -411,71 +434,65 @@ export class VMExecution extends Execution { }*/ // Option a): set iterator head to the parent block so that an // error can repeatedly processed for debugging - const { number } = block.header - const hash = short(block.hash()) + const { number } = errorBlock.header + const hash = short(errorBlock.hash()) this.config.logger.warn( `Execution of block number=${number} hash=${hash} hardfork=${this.hardfork} failed:\n${error}` ) if (this.config.debugCode) { - await debugCodeReplayBlock(this, block) + await debugCodeReplayBlock(this, errorBlock) } this.config.events.emit(Event.SYNC_EXECUTION_VM_ERROR, error) - errorBlock = block + const actualExecuted = Number(errorBlock.header.number - startHeadBlock.header.number) + return actualExecuted + } else { + this.config.logger.error(`VM execution failed with error`, error) + return null } - }, - this.config.numBlocksPerIteration, - // release lock on this callback so other blockchain ops can happen while this block is being executed - true - ) - numExecuted = await this.vmPromise + }) - // TODO: one should update the iterator head later as this is dangerous for the blockchain and can cause - // problems in concurrent execution - if (errorBlock !== undefined) { - await this.chain.blockchain.setIteratorHead( - 'vm', - (errorBlock as unknown as Block).header.parentHash - ) - return 0 - } - let endHeadBlock - if (typeof this.vm.blockchain.getIteratorHead === 'function') { - endHeadBlock = await this.vm.blockchain.getIteratorHead('vm') - } else { - throw new Error('cannot get iterator head: blockchain has no getIteratorHead function') - } + numExecuted = await this.vmPromise + if (numExecuted !== null) { + let endHeadBlock + if (typeof this.vm.blockchain.getIteratorHead === 'function') { + endHeadBlock = await this.vm.blockchain.getIteratorHead('vm') + } else { + throw new Error('cannot get iterator head: blockchain has no getIteratorHead function') + } - if (typeof numExecuted === 'number' && numExecuted > 0) { - const firstNumber = startHeadBlock.header.number - const firstHash = short(startHeadBlock.hash()) - const lastNumber = endHeadBlock.header.number - const lastHash = short(endHeadBlock.hash()) - const baseFeeAdd = - this.config.execCommon.gteHardfork(Hardfork.London) === true - ? `baseFee=${endHeadBlock.header.baseFeePerGas} ` - : '' - const tdAdd = - this.config.execCommon.gteHardfork(Hardfork.Paris) === true - ? '' - : `td=${this.chain.blocks.td} ` - this.config.logger.info( - `Executed blocks count=${numExecuted} first=${firstNumber} hash=${firstHash} ${tdAdd}${baseFeeAdd}hardfork=${this.hardfork} last=${lastNumber} hash=${lastHash} txs=${txCounter}` - ) - } else { - this.config.logger.debug( - `No blocks executed past chain head hash=${short(endHeadBlock.hash())} number=${ - endHeadBlock.header.number - }` - ) - } - startHeadBlock = endHeadBlock - if (typeof this.vm.blockchain.getCanonicalHeadBlock !== 'function') { - throw new Error( - 'cannot get iterator head: blockchain has no getCanonicalHeadBlock function' - ) + if (typeof numExecuted === 'number' && numExecuted > 0) { + const firstNumber = startHeadBlock.header.number + const firstHash = short(startHeadBlock.hash()) + const lastNumber = endHeadBlock.header.number + const lastHash = short(endHeadBlock.hash()) + const baseFeeAdd = + this.config.execCommon.gteHardfork(Hardfork.London) === true + ? `baseFee=${endHeadBlock.header.baseFeePerGas} ` + : '' + const tdAdd = + this.config.execCommon.gteHardfork(Hardfork.Paris) === true + ? '' + : `td=${this.chain.blocks.td} ` + this.config.logger.info( + `Executed blocks count=${numExecuted} first=${firstNumber} hash=${firstHash} ${tdAdd}${baseFeeAdd}hardfork=${this.hardfork} last=${lastNumber} hash=${lastHash} txs=${txCounter}` + ) + } else { + this.config.logger.debug( + `No blocks executed past chain head hash=${short(endHeadBlock.hash())} number=${ + endHeadBlock.header.number + }` + ) + } + startHeadBlock = endHeadBlock + if (typeof this.vm.blockchain.getCanonicalHeadBlock !== 'function') { + throw new Error( + 'cannot get iterator head: blockchain has no getCanonicalHeadBlock function' + ) + } + canonicalHead = await this.vm.blockchain.getCanonicalHeadBlock() } - canonicalHead = await this.vm.blockchain.getCanonicalHeadBlock() } + this.running = false return numExecuted ?? 0 } diff --git a/packages/client/lib/util/debug.ts b/packages/client/lib/util/debug.ts index 00d9702ed3..c072b0b020 100644 --- a/packages/client/lib/util/debug.ts +++ b/packages/client/lib/util/debug.ts @@ -6,7 +6,7 @@ import type { VMExecution } from '../execution' import type { Block } from '@ethereumjs/block' /** - * Generates a code snippet which can be used to replay an erraneous block + * Generates a code snippet which can be used to replay an erroneous block * locally in the VM * * @param block