Merge remote-tracking branch 'origin/master' into feature/rpc-state-o…

…verride

.github/workflows/build-solutions.yml

@@ -5,6 +5,7 @@ on:
     branches: [master]
   push:
     branches: [master]
+  merge_group:
 
 jobs:
   build:

.github/workflows/sync-supported-chains.yml

@@ -232,6 +232,7 @@ jobs:
         with:
           app-id: ${{ vars.APP_ID }}
           private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: "nethermind,post-merge-smoke-tests"
 
       - name: Trigger notification action on test repo
         if: github.event.workflow_run.head_branch == 'master' || startsWith(github.event.workflow_run.head_branch, 'release/')

.github/workflows/test-assertoor.yml

@@ -68,9 +68,7 @@ jobs:
         fi
 
         if 
-        [[ "$ref" == "refs/heads/master" ]] && 
-        [[ "${{ steps.extract_dockerfile.outputs.dockerfile }}" == "Dockerfile" ]] && 
-        [[ "${{ steps.extract_dockerfile.outputs.build-config }}" == "release" ]]; then
+        [[ "$ref" == "refs/heads/master" ]]; then
           echo "skip_docker_build=true" >> $GITHUB_OUTPUT
         else
           echo "skip_docker_build=false" >> $GITHUB_OUTPUT

src/Nethermind/Directory.Packages.props

@@ -66,7 +66,7 @@
     <PackageVersion Include="Pyroscope" Version="0.8.14" />
     <PackageVersion Include="ReadLine" Version="2.0.1" />
     <PackageVersion Include="RichardSzalay.MockHttp" Version="7.0.0" />
-    <PackageVersion Include="RocksDB" Version="8.10.0.45817" />
+    <PackageVersion Include="RocksDB" Version="9.4.0.50294" />
     <PackageVersion Include="SCrypt" Version="2.0.0.2" />
     <PackageVersion Include="Shouldly" Version="4.2.1" />
     <PackageVersion Include="Snappier" Version="1.1.6" />

src/Nethermind/Nethermind.Config/ExitCodes.cs

@@ -15,8 +15,9 @@ public static class ExitCodes
     public const int TooLongExtraData = 102;
     public const int ConflictingConfigurations = 103;
     public const int LowDiskSpace = 104;
-    public const int DuplicatedArguments = 105;
-    public const int UnrecognizedArgument = 106;
+    public const int DuplicatedOption = 105;
+    public const int UnrecognizedOption = 106;
+    public const int ForbiddenOptionValue = 107;
 
     // Posix exit code
     // https://tldp.org/LDP/abs/html/exitcodes.html

src/Nethermind/Nethermind.Consensus/Processing/BlockCachePreWarmer.cs

@@ -48,6 +48,8 @@ public Task PreWarmCaches(Block suggestedBlock, Hash256? parentStateRoot, Cancel
 
     public void ClearCaches() => targetWorldState?.ClearCache();
 
+    public Task ClearCachesInBackground() => targetWorldState?.ClearCachesInBackground() ?? Task.CompletedTask;
+
     private void PreWarmCachesParallel(Block suggestedBlock, Hash256 parentStateRoot, CancellationToken cancellationToken)
     {
         if (cancellationToken.IsCancellationRequested) return;

src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.cs

@@ -51,6 +51,7 @@ public partial class BlockProcessor : IBlockProcessor
     /// to any block-specific tracers.
     /// </summary>
     protected BlockReceiptsTracer ReceiptsTracer { get; set; }
+    private readonly Func<Task, Task> _clearCaches;
 
     public BlockProcessor(
         ISpecProvider? specProvider,
@@ -78,6 +79,7 @@ public BlockProcessor(
         _preWarmer = preWarmer;
         _beaconBlockRootHandler = new BeaconBlockRootHandler();
         ReceiptsTracer = new BlockReceiptsTracer();
+        _clearCaches = _ => _preWarmer.ClearCachesInBackground();
     }
 
     public event EventHandler<BlockProcessedEventArgs>? BlockProcessed;
@@ -124,6 +126,10 @@ the previous head state.*/
                     : _preWarmer?.PreWarmCaches(suggestedBlock, preBlockStateRoot!, cancellationTokenSource.Token);
                 (Block processedBlock, TxReceipt[] receipts) = ProcessOne(suggestedBlock, options, blockTracer, stateOverride);
                 // Block is processed, we can cancel the prewarm task
+                if (preWarmTask is not null)
+                {
+                    preWarmTask = preWarmTask.ContinueWith(_clearCaches).Unwrap();
+                }
                 cancellationTokenSource.Cancel();
                 processedBlocks[i] = processedBlock;
 

src/Nethermind/Nethermind.Consensus/Processing/IBlockCachePreWarmer.cs

@@ -12,4 +12,5 @@ public interface IBlockCachePreWarmer
 {
     Task PreWarmCaches(Block suggestedBlock, Hash256 parentStateRoot, CancellationToken cancellationToken = default);
     void ClearCaches();
+    Task ClearCachesInBackground();
 }

src/Nethermind/Nethermind.Core.Test/Collections/LockableConcurrentDictionaryTests.cs → src/Nethermind/Nethermind.Core.Test/Collections/ConcurrentDictionaryTests.cs

@@ -10,7 +10,7 @@
 
 namespace Nethermind.Core.Test.Collections;
 
-public class LockableConcurrentDictionaryTests
+public class ConcurrentDictionaryTests
 {
     [Test]
     public void Locks()
@@ -28,4 +28,14 @@ public void Locks()
         updateTask.Wait();
         dictionary.ContainsKey(3).Should().BeTrue();
     }
+
+    [Test]
+    public void NoResizeClear()
+    {
+        // Tests that the reflection works
+        ConcurrentDictionary<int, int> dictionary = new(new Dictionary<int, int> { { 0, 0 }, { 1, 1 }, { 2, 2 } });
+        dictionary.NoResizeClear();
+
+        dictionary.Count.Should().Be(0);
+    }
 }

src/Nethermind/Nethermind.Core/Caching/ClockCache.cs

@@ -7,14 +7,17 @@
 using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
 
+using Nethermind.Core.Collections;
 using Nethermind.Core.Threading;
 
+using CollectionExtensions = Nethermind.Core.Collections.CollectionExtensions;
+
 namespace Nethermind.Core.Caching;
 
 public sealed class ClockCache<TKey, TValue>(int maxCapacity) : ClockCacheBase<TKey>(maxCapacity)
     where TKey : struct, IEquatable<TKey>
 {
-    private readonly ConcurrentDictionary<TKey, LruCacheItem> _cacheMap = new ConcurrentDictionary<TKey, LruCacheItem>();
+    private readonly ConcurrentDictionary<TKey, LruCacheItem> _cacheMap = new(CollectionExtensions.LockPartitions, maxCapacity);
     private readonly McsLock _lock = new();
 
     public TValue Get(TKey key)
@@ -154,7 +157,7 @@ public bool Delete(TKey key)
         using var lockRelease = _lock.Acquire();
 
         base.Clear();
-        _cacheMap.Clear();
+        _cacheMap.NoResizeClear();
     }
 
     public bool Contains(TKey key)

src/Nethermind/Nethermind.Core/Caching/ClockKeyCache.cs

@@ -7,14 +7,17 @@
 using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
 
+using Nethermind.Core.Collections;
 using Nethermind.Core.Threading;
 
+using CollectionExtensions = Nethermind.Core.Collections.CollectionExtensions;
+
 namespace Nethermind.Core.Caching;
 
 public sealed class ClockKeyCache<TKey>(int maxCapacity) : ClockCacheBase<TKey>(maxCapacity)
     where TKey : struct, IEquatable<TKey>
 {
-    private readonly ConcurrentDictionary<TKey, int> _cacheMap = new ConcurrentDictionary<TKey, int>();
+    private readonly ConcurrentDictionary<TKey, int> _cacheMap = new(CollectionExtensions.LockPartitions, maxCapacity);
     private readonly McsLock _lock = new();
 
     public bool Get(TKey key)
@@ -128,7 +131,7 @@ public bool Delete(TKey key)
         using var lockRelease = _lock.Acquire();
 
         base.Clear();
-        _cacheMap.Clear();
+        _cacheMap.NoResizeClear();
     }
 
     public bool Contains(TKey key)

src/Nethermind/Nethermind.Core/Collections/CollectionExtensions.cs

@@ -1,12 +1,19 @@
 // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
 // SPDX-License-Identifier: LGPL-3.0-only
 
+using System;
+using System.Collections;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
+using System.Linq.Expressions;
+using System.Reflection;
 
 namespace Nethermind.Core.Collections
 {
     public static class CollectionExtensions
     {
+        public static int LockPartitions { get; } = Environment.ProcessorCount * 16;
+
         public static void AddRange<T>(this ICollection<T> list, IEnumerable<T> items)
         {
             foreach (T item in items)
@@ -22,5 +29,62 @@ public static void AddRange<T>(this ICollection<T> list, params T[] items)
                 list.Add(items[index]);
             }
         }
+
+        public static bool NoResizeClear<TKey, TValue>(this ConcurrentDictionary<TKey, TValue>? dictionary)
+                where TKey : notnull
+        {
+            if (dictionary is null || dictionary.IsEmpty)
+            {
+                return false;
+            }
+
+            using var handle = dictionary.AcquireLock();
+
+            ClearCache<TKey, TValue>.Clear(dictionary);
+            return true;
+        }
+
+        private static class ClearCache<TKey, TValue> where TKey : notnull
+        {
+            public static readonly Action<ConcurrentDictionary<TKey, TValue>> Clear = CreateNoResizeClearExpression();
+
+            private static Action<ConcurrentDictionary<TKey, TValue>> CreateNoResizeClearExpression()
+            {
+                // Parameters
+                var dictionaryParam = Expression.Parameter(typeof(ConcurrentDictionary<TKey, TValue>), "dictionary");
+
+                // Access _tables field
+                var tablesField = typeof(ConcurrentDictionary<TKey, TValue>).GetField("_tables", BindingFlags.NonPublic | BindingFlags.Instance);
+                var tablesAccess = Expression.Field(dictionaryParam, tablesField!);
+
+                // Access _buckets and _countPerLock fields
+                var tablesType = tablesField!.FieldType;
+                var bucketsField = tablesType.GetField("_buckets", BindingFlags.NonPublic | BindingFlags.Instance);
+                var countPerLockField = tablesType.GetField("_countPerLock", BindingFlags.NonPublic | BindingFlags.Instance);
+
+                var bucketsAccess = Expression.Field(tablesAccess, bucketsField!);
+                var countPerLockAccess = Expression.Field(tablesAccess, countPerLockField!);
+
+                // Clear arrays using Array.Clear
+                var clearMethod = typeof(Array).GetMethod("Clear", new[] { typeof(Array), typeof(int), typeof(int) });
+
+                var clearBuckets = Expression.Call(clearMethod!,
+                    bucketsAccess,
+                    Expression.Constant(0),
+                    Expression.ArrayLength(bucketsAccess));
+
+                var clearCountPerLock = Expression.Call(clearMethod!,
+                    countPerLockAccess,
+                    Expression.Constant(0),
+                    Expression.ArrayLength(countPerLockAccess));
+
+                // Block to execute both clears
+                var block = Expression.Block(clearBuckets, clearCountPerLock);
+
+                // Compile the expression into a lambda
+                return Expression.Lambda<Action<ConcurrentDictionary<TKey, TValue>>>(block, dictionaryParam).Compile();
+            }
+        }
     }
 }
+

src/Nethermind/Nethermind.Db.Test/RocksDbTests.cs

@@ -11,5 +11,5 @@ namespace Nethermind.Db.Test;
 internal static class RocksDbTests
 {
     [Test]
-    public static void Should_have_required_version() => DbOnTheRocks.GetRocksDbVersion().Should().Be("8.10.0");
+    public static void Should_have_required_version() => DbOnTheRocks.GetRocksDbVersion().Should().Be("9.4.0");
 }