Merge branch 'master' into feature/op-cl

.editorconfig

@@ -9,7 +9,7 @@ indent_style = space
 insert_final_newline = true
 trim_trailing_whitespace = true
 
-file_header_template = SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited\nSPDX-License-Identifier: LGPL-3.0-only
+file_header_template = SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited\nSPDX-License-Identifier: LGPL-3.0-only
 
 [*.cs]
 indent_size = 4

.github/workflows/codeql.yml

@@ -20,25 +20,20 @@ jobs:
       matrix:
         language: ['csharp']
     steps:
-      - name: Free up disk space
-        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be #v1.3.1
-        with:
-          large-packages: false
-          tool-cache: false
       - name: Check out repository
-        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 #v4.2.0
-        with:
-          submodules: true
+        uses: actions/checkout@v4
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@5618c9fc1e675841ca52c1c6b1304f5255a905a0 #v2.19.0
+        uses: github/codeql-action/init@v3
         with:
           languages: ${{ matrix.language }}
+          queries: security-and-quality
+          packs: githubsecuritylab/codeql-${{ matrix.language }}-queries
       - name: Set up .NET
-        uses: actions/setup-dotnet@6bd8b7f7774af54e05809fcc5431931b3eb1ddee #v4.0.1
+        uses: actions/setup-dotnet@v4
       - name: Build Nethermind
         working-directory: src/Nethermind
         run: dotnet build Nethermind.sln -c release
       - name: Perform CodeQL analysis
-        uses: github/codeql-action/analyze@5618c9fc1e675841ca52c1c6b1304f5255a905a0 #v2.19.0
+        uses: github/codeql-action/analyze@v3
         with:
           category: '/language:${{ matrix.language }}'

CONTRIBUTING.md

@@ -59,7 +59,7 @@ Branch names must follow the `kebab-case` or `snake_case` pattern and be all low
 The following notice must be included as a header in all source files if possible.
 
 ```
-// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited
+// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited
 // SPDX-License-Identifier: LGPL-3.0-only
 ```
 

src/Nethermind/Nethermind.Api/NethermindApi.cs

@@ -76,11 +76,10 @@ public NethermindApi(IConfigProvider configProvider, IJsonSerializer jsonSeriali
         public IBlockchainBridge CreateBlockchainBridge()
         {
             ReadOnlyBlockTree readOnlyTree = BlockTree!.AsReadOnly();
-            OverridableWorldStateManager overridableWorldStateManager = new(DbProvider!, WorldStateManager!.TrieStore, LogManager);
 
             // TODO: reuse the same trie cache here
             OverridableTxProcessingEnv txProcessingEnv = new(
-                overridableWorldStateManager,
+                WorldStateManager!.CreateOverridableWorldScope(),
                 readOnlyTree,
                 SpecProvider!,
                 LogManager);

src/Nethermind/Nethermind.Clique.Test/CliqueBlockProducerTests.cs

@@ -380,8 +380,8 @@ public On AssertVote(PrivateKey nodeKey, long number, Address address, bool vote
         {
             WaitForNumber(nodeKey, number);
             if (_logger.IsInfo) _logger.Info($"ASSERTING {vote} VOTE ON {address} AT BLOCK {number}");
-            Assert.That(_blockTrees[nodeKey].FindBlock(number, BlockTreeLookupOptions.None).Header.Nonce, Is.EqualTo(vote ? Consensus.Clique.Clique.NonceAuthVote : Consensus.Clique.Clique.NonceDropVote), nodeKey + " vote nonce");
-            Assert.That(_blockTrees[nodeKey].FindBlock(number, BlockTreeLookupOptions.None).Beneficiary, Is.EqualTo(address), nodeKey.Address + " vote nonce");
+            Assert.That(() => _blockTrees[nodeKey].FindBlock(number, BlockTreeLookupOptions.None)?.Header.Nonce, Is.EqualTo(vote ? Consensus.Clique.Clique.NonceAuthVote : Consensus.Clique.Clique.NonceDropVote).After(_timeout, 100), nodeKey + " vote nonce");
+            Assert.That(() => _blockTrees[nodeKey].FindBlock(number, BlockTreeLookupOptions.None)?.Beneficiary, Is.EqualTo(address).After(_timeout, 100), nodeKey.Address + " vote nonce");
             return this;
         }
 
@@ -561,7 +561,7 @@ public On Wait(int i)
         }
     }
 
-    private static readonly int _timeout = 2000; // this has to cover block period of second + wiggle of up to 500ms * (signers - 1) + 100ms delay of the block readiness check
+    private static readonly int _timeout = 5000; // this has to cover block period of second + wiggle of up to 500ms * (signers - 1) + 100ms delay of the block readiness check
 
     [Test]
     public async Task Can_produce_block_with_transactions()

src/Nethermind/Nethermind.Config/JsonConfigSource.cs

@@ -12,6 +12,8 @@ namespace Nethermind.Config;
 
 public class JsonConfigSource : IConfigSource
 {
+    private const string SchemaKey = "$schema";
+
     public JsonConfigSource(string configFilePath)
     {
         LoadJsonConfig(configFilePath);
@@ -22,7 +24,7 @@ private void ApplyJsonConfig(string jsonContent)
         try
         {
             using var json = JsonDocument.Parse(jsonContent);
-            foreach (var moduleEntry in json.RootElement.EnumerateObject())
+            foreach (var moduleEntry in json.RootElement.EnumerateObject().Where(o => o.Name != SchemaKey))
             {
                 LoadModule(moduleEntry.Name, moduleEntry.Value);
             }
@@ -72,7 +74,7 @@ private void LoadModule(string moduleName, JsonElement configItems)
     {
         var itemsDict = new Dictionary<string, string>(StringComparer.InvariantCultureIgnoreCase);
 
-        foreach (var configItem in configItems.EnumerateObject())
+        foreach (var configItem in configItems.EnumerateObject().Where(o => o.Name != SchemaKey))
         {
             var key = configItem.Name;
             if (!itemsDict.ContainsKey(key))

src/Nethermind/Nethermind.Consensus.AuRa/InitializationSteps/RegisterAuRaRpcModules.cs

@@ -66,8 +66,7 @@ protected override void RegisterTraceRpcModule(IRpcModuleProvider rpcModuleProvi
         StepDependencyException.ThrowIfNull(_api.SpecProvider);
 
         AuRaTraceModuleFactory traceModuleFactory = new(
-            _api.WorldStateManager.TrieStore,
-            _api.DbProvider,
+            _api.WorldStateManager,
             _api.BlockTree,
             _jsonRpcConfig,
             _api.BlockPreprocessor,
@@ -82,8 +81,7 @@ protected override void RegisterTraceRpcModule(IRpcModuleProvider rpcModuleProvi
     }
 
     protected class AuRaTraceModuleFactory(
-        IReadOnlyTrieStore trieStore,
-        IDbProvider dbProvider,
+        IWorldStateManager worldStateManager,
         IBlockTree blockTree,
         IJsonRpcConfig jsonRpcConfig,
         IBlockPreprocessorStep recoveryStep,
@@ -93,10 +91,10 @@ protected class AuRaTraceModuleFactory(
         IPoSSwitcher poSSwitcher,
         ILogManager logManager,
         IAuRaBlockProcessorFactory factory)
-        : TraceModuleFactory(trieStore, dbProvider, blockTree, jsonRpcConfig, recoveryStep, rewardCalculatorSource,
+        : TraceModuleFactory(worldStateManager, blockTree, jsonRpcConfig, recoveryStep, rewardCalculatorSource,
             receiptFinder, specProvider, poSSwitcher, logManager)
     {
-        protected override ReadOnlyChainProcessingEnv CreateChainProcessingEnv(OverridableWorldStateManager worldStateManager,
+        protected override ReadOnlyChainProcessingEnv CreateChainProcessingEnv(IOverridableWorldScope worldStateManager,
             IBlockProcessor.IBlockTransactionsExecutor transactionsExecutor, IReadOnlyTxProcessingScope scope,
             IRewardCalculator rewardCalculator)
         {
@@ -202,7 +200,7 @@ protected override void RegisterDebugRpcModule(IRpcModuleProvider rpcModuleProvi
         StepDependencyException.ThrowIfNull(_api.SpecProvider);
 
         AuRaDebugModuleFactory debugModuleFactory = new(
-            _api.WorldStateManager.TrieStore,
+            _api.WorldStateManager,
             _api.DbProvider,
             _api.BlockTree,
             _jsonRpcConfig,
@@ -223,7 +221,7 @@ protected override void RegisterDebugRpcModule(IRpcModuleProvider rpcModuleProvi
     }
 
     protected class AuRaDebugModuleFactory(
-        IReadOnlyTrieStore trieStore,
+        IWorldStateManager worldStateManager,
         IDbProvider dbProvider,
         IBlockTree blockTree,
         IJsonRpcConfig jsonRpcConfig,
@@ -239,12 +237,12 @@ protected class AuRaDebugModuleFactory(
         IFileSystem fileSystem,
         ILogManager logManager,
         IAuRaBlockProcessorFactory factory)
-        : DebugModuleFactory(trieStore, dbProvider, blockTree, jsonRpcConfig, blockValidator, recoveryStep,
+        : DebugModuleFactory(worldStateManager, dbProvider, blockTree, jsonRpcConfig, blockValidator, recoveryStep,
             rewardCalculator, receiptStorage, receiptsMigration, configProvider, specProvider, syncModeSelector,
             badBlockStore, fileSystem, logManager)
     {
         protected override ReadOnlyChainProcessingEnv CreateReadOnlyChainProcessingEnv(IReadOnlyTxProcessingScope scope,
-            OverridableWorldStateManager worldStateManager, BlockProcessor.BlockValidationTransactionsExecutor transactionsExecutor)
+            IOverridableWorldScope worldStateManager, BlockProcessor.BlockValidationTransactionsExecutor transactionsExecutor)
         {
             return new AuRaReadOnlyChainProcessingEnv(
                 scope,

src/Nethermind/Nethermind.Consensus/Processing/BlockCachePreWarmer.cs

@@ -24,7 +24,6 @@ namespace Nethermind.Consensus.Processing;
 public sealed class BlockCachePreWarmer(ReadOnlyTxProcessingEnvFactory envFactory, ISpecProvider specProvider, ILogManager logManager, PreBlockCaches? preBlockCaches = null) : IBlockCachePreWarmer
 {
     private readonly ObjectPool<IReadOnlyTxProcessorSource> _envPool = new DefaultObjectPool<IReadOnlyTxProcessorSource>(new ReadOnlyTxProcessingEnvPooledObjectPolicy(envFactory), Environment.ProcessorCount * 4);
-    private readonly ObjectPool<SystemTransaction> _systemTransactionPool = new DefaultObjectPool<SystemTransaction>(new DefaultPooledObjectPolicy<SystemTransaction>(), Environment.ProcessorCount * 4);
     private readonly ILogger _logger = logManager.GetClassLogger<BlockCachePreWarmer>();
 
     public Task PreWarmCaches(Block suggestedBlock, Hash256? parentStateRoot, IReleaseSpec spec, CancellationToken cancellationToken = default, params ReadOnlySpan<IHasAccessList> systemAccessLists)
@@ -98,18 +97,18 @@ private void WarmupWithdrawals(ParallelOptions parallelOptions, IReleaseSpec spe
         {
             if (spec.WithdrawalsEnabled && block.Withdrawals is not null)
             {
-                ParallelUnbalancedWork.For(0, block.Withdrawals.Length, parallelOptions, (preWarmer: this, block, stateRoot),
+                ParallelUnbalancedWork.For(0, block.Withdrawals.Length, parallelOptions, (envPool: _envPool, block, stateRoot),
                     static (i, state) =>
                     {
-                        IReadOnlyTxProcessorSource env = state.preWarmer._envPool.Get();
+                        IReadOnlyTxProcessorSource env = state.envPool.Get();
                         try
                         {
                             using IReadOnlyTxProcessingScope scope = env.Build(state.stateRoot);
                             scope.WorldState.WarmUp(state.block.Withdrawals[i].Address);
                         }
                         finally
                         {
-                            state.preWarmer._envPool.Return(env);
+                            state.envPool.Return(env);
                         }
 
                         return state;
@@ -135,15 +134,13 @@ private void WarmupTransactions(ParallelOptions parallelOptions, IReleaseSpec sp
             ParallelUnbalancedWork.For<BlockState>(0, block.Transactions.Length, parallelOptions, new(this, block, stateRoot, spec), static (i, state) =>
             {
                 IReadOnlyTxProcessorSource env = state.PreWarmer._envPool.Get();
-                SystemTransaction systemTransaction = state.PreWarmer._systemTransactionPool.Get();
                 Transaction? tx = null;
                 try
                 {
                     // If the transaction has already been processed or being processed, exit early
                     if (state.Block.TransactionProcessed > i) return state;
 
                     tx = state.Block.Transactions[i];
-                    tx.CopyTo(systemTransaction);
                     using IReadOnlyTxProcessingScope scope = env.Build(state.StateRoot);
 
                     Address senderAddress = tx.SenderAddress!;
@@ -170,7 +167,7 @@ private void WarmupTransactions(ParallelOptions parallelOptions, IReleaseSpec sp
                     {
                         scope.WorldState.WarmUp(tx.AccessList); // eip-2930
                     }
-                    TransactionResult result = scope.TransactionProcessor.Warmup(systemTransaction, new BlockExecutionContext(state.Block.Header.Clone()), NullTxTracer.Instance);
+                    TransactionResult result = scope.TransactionProcessor.Warmup(tx, new BlockExecutionContext(state.BlockHeader), NullTxTracer.Instance);
                     if (state.PreWarmer._logger.IsTrace) state.PreWarmer._logger.Trace($"Finished pre-warming cache for tx[{i}] {tx.Hash} with {result}");
                 }
                 catch (Exception ex) when (ex is EvmException or OverflowException)
@@ -183,7 +180,6 @@ private void WarmupTransactions(ParallelOptions parallelOptions, IReleaseSpec sp
                 }
                 finally
                 {
-                    state.PreWarmer._systemTransactionPool.Return(systemTransaction);
                     state.PreWarmer._envPool.Return(env);
                 }
 
@@ -246,11 +242,12 @@ private void WarmupAddresses(ParallelOptions parallelOptions, Block block)
         {
             if (parallelOptions.CancellationToken.IsCancellationRequested) return;
 
+            ObjectPool<IReadOnlyTxProcessorSource> envPool = PreWarmer._envPool;
             try
             {
                 if (SystemTxAccessLists is not null)
                 {
-                    var env = PreWarmer._envPool.Get();
+                    var env = envPool.Get();
                     try
                     {
                         using IReadOnlyTxProcessingScope scope = env.Build(StateRoot);
@@ -262,38 +259,36 @@ private void WarmupAddresses(ParallelOptions parallelOptions, Block block)
                     }
                     finally
                     {
-                        PreWarmer._envPool.Return(env);
+                        envPool.Return(env);
                         SystemTxAccessLists.Dispose();
                     }
                 }
 
-                ParallelUnbalancedWork.For(0, block.Transactions.Length, parallelOptions, (preWarmer: PreWarmer, block, StateRoot),
+                AddressWarmingState baseState = new(envPool, block, StateRoot);
+
+                ParallelUnbalancedWork.For(
+                    0,
+                    block.Transactions.Length,
+                    parallelOptions,
+                    baseState.InitThreadState,
                 static (i, state) =>
                 {
-                    Transaction tx = state.block.Transactions[i];
+                    Transaction tx = state.Block.Transactions[i];
                     Address? sender = tx.SenderAddress;
 
-                    var env = state.preWarmer._envPool.Get();
-                    try
+                    if (sender is not null)
                     {
-                        using IReadOnlyTxProcessingScope scope = env.Build(state.StateRoot);
-                        if (sender is not null)
-                        {
-                            scope.WorldState.WarmUp(sender);
-                        }
-                        Address to = tx.To;
-                        if (to is not null)
-                        {
-                            scope.WorldState.WarmUp(to);
-                        }
+                        state.Scope.WorldState.WarmUp(sender);
                     }
-                    finally
+                    Address to = tx.To;
+                    if (to is not null)
                     {
-                        state.preWarmer._envPool.Return(env);
+                        state.Scope.WorldState.WarmUp(to);
                     }
 
                     return state;
-                });
+                },
+                AddressWarmingState.FinallyAction);
             }
             catch (OperationCanceledException)
             {
@@ -302,18 +297,50 @@ private void WarmupAddresses(ParallelOptions parallelOptions, Block block)
         }
     }
 
+    private readonly struct AddressWarmingState(ObjectPool<IReadOnlyTxProcessorSource> envPool, Block block, Hash256 stateRoot) : IDisposable
+    {
+        public static Action<AddressWarmingState> FinallyAction { get; } = DisposeThreadState;
+
+        public readonly ObjectPool<IReadOnlyTxProcessorSource> EnvPool = envPool;
+        public readonly Block Block = block;
+        public readonly Hash256 StateRoot = stateRoot;
+        public readonly IReadOnlyTxProcessorSource? Env;
+        public readonly IReadOnlyTxProcessingScope? Scope;
+
+        public AddressWarmingState(ObjectPool<IReadOnlyTxProcessorSource> envPool, Block block, Hash256 stateRoot, IReadOnlyTxProcessorSource env, IReadOnlyTxProcessingScope scope) : this(envPool, block, stateRoot)
+        {
+            Env = env;
+            Scope = scope;
+        }
+
+        public AddressWarmingState InitThreadState()
+        {
+            IReadOnlyTxProcessorSource env = EnvPool.Get();
+            return new(EnvPool, Block, StateRoot, env, scope: env.Build(StateRoot));
+        }
+
+        public void Dispose()
+        {
+            Scope.Dispose();
+            EnvPool.Return(Env);
+        }
+
+        private static void DisposeThreadState(AddressWarmingState state) => state.Dispose();
+    }
+
     private class ReadOnlyTxProcessingEnvPooledObjectPolicy(ReadOnlyTxProcessingEnvFactory envFactory) : IPooledObjectPolicy<IReadOnlyTxProcessorSource>
     {
         public IReadOnlyTxProcessorSource Create() => envFactory.Create();
         public bool Return(IReadOnlyTxProcessorSource obj) => true;
     }
 
-    private struct BlockState(BlockCachePreWarmer preWarmer, Block block, Hash256 stateRoot, IReleaseSpec spec)
+    private readonly struct BlockState(BlockCachePreWarmer preWarmer, Block block, Hash256 stateRoot, IReleaseSpec spec)
     {
-        public BlockCachePreWarmer PreWarmer = preWarmer;
-        public Block Block = block;
-        public Hash256 StateRoot = stateRoot;
-        public IReleaseSpec Spec = spec;
+        public readonly BlockCachePreWarmer PreWarmer = preWarmer;
+        public readonly Block Block = block;
+        public readonly Hash256 StateRoot = stateRoot;
+        public readonly IReleaseSpec Spec = spec;
+        public readonly BlockHeader BlockHeader => Block.Header;
     }
 }