Merge branch 'master' into 7746_update_jwt_path

src/Nethermind/Nethermind.Api/IApiWithNetwork.cs

@@ -47,6 +47,7 @@ public interface IApiWithNetwork : IApiWithBlockchain
         ISynchronizer? Synchronizer { get; }
         ISyncModeSelector SyncModeSelector { get; }
         ISyncProgressResolver? SyncProgressResolver { get; }
+        ISyncPointers? SyncPointers { get; }
         IPivot? Pivot { get; set; }
         ISyncPeerPool? SyncPeerPool { get; }
         IPeerDifficultyRefreshPool? PeerDifficultyRefreshPool { get; }

src/Nethermind/Nethermind.Api/IBasicApi.cs

@@ -7,6 +7,7 @@
 using Autofac;
 using Nethermind.Abi;
 using Nethermind.Api.Extensions;
+using Nethermind.Blockchain.Receipts;
 using Nethermind.Blockchain.Synchronization;
 using Nethermind.Config;
 using Nethermind.Core;
@@ -63,6 +64,7 @@ public ContainerBuilder ConfigureContainerBuilderFromBasicApi(ContainerBuilder b
             builder
                 .AddPropertiesFrom<IBasicApi>(this)
                 .AddSingleton(ConfigProvider.GetConfig<ISyncConfig>())
+                .AddSingleton(ConfigProvider.GetConfig<IReceiptConfig>())
                 .AddModule(new DbModule());
 
             return builder;

src/Nethermind/Nethermind.Api/NethermindApi.cs

@@ -190,6 +190,7 @@ public ISealEngine SealEngine
         public ISyncModeSelector SyncModeSelector => ApiWithNetworkServiceContainer?.Resolve<ISyncModeSelector>()!;
 
         public ISyncProgressResolver? SyncProgressResolver => ApiWithNetworkServiceContainer?.Resolve<ISyncProgressResolver>();
+        public ISyncPointers? SyncPointers => ApiWithNetworkServiceContainer?.Resolve<ISyncPointers>();
         public IBetterPeerStrategy? BetterPeerStrategy { get; set; }
         public IPivot? Pivot { get; set; }
         public ISyncPeerPool? SyncPeerPool => ApiWithNetworkServiceContainer?.Resolve<ISyncPeerPool>();

src/Nethermind/Nethermind.Blockchain.Test/BlockTreeTests.cs

@@ -1170,47 +1170,6 @@ public void Loads_lowest_inserted_header_correctly(long beginIndex, long inserte
         Assert.That(loadedTree.LowestInsertedHeader?.Number, Is.EqualTo(expectedResult), "loaded tree");
     }
 
-    [Test, MaxTime(Timeout.MaxTestTime), TestCaseSource(nameof(SourceOfBSearchTestCases))]
-    public void Loads_lowest_inserted_body_correctly(long beginIndex, long insertedBlocks)
-    {
-        // left old code to prove that it does not matter for the result nowadays
-        // we store and no longer binary search lowest body number
-
-        MemDb blocksDb = new();
-        blocksDb.Set(BlockTree.LowestInsertedBodyNumberDbEntryAddress, Rlp.Encode(1L).Bytes);
-
-        SyncConfig syncConfig = new()
-        {
-            PivotNumber = beginIndex.ToString(),
-        };
-
-        BlockTreeBuilder builder = Build.A.BlockTree()
-            .WithBlocksDb(blocksDb)
-            .WithSyncConfig(syncConfig)
-            .WithoutSettingHead;
-
-        BlockTree tree = builder.TestObject;
-
-        tree.SuggestBlock(Build.A.Block.Genesis.TestObject);
-
-        for (long i = beginIndex; i > beginIndex - insertedBlocks; i--)
-        {
-            Block block = Build.A.Block.WithNumber(i).WithTotalDifficulty(i).TestObject;
-            tree.Insert(block.Header);
-            tree.Insert(block);
-        }
-
-        BlockTree loadedTree = Build.A.BlockTree()
-            .WithoutSettingHead
-            .WithDatabaseFrom(builder)
-            .WithSyncConfig(syncConfig)
-            .TestObject;
-
-        Assert.That(tree.LowestInsertedBodyNumber, Is.EqualTo(null), "tree");
-        Assert.That(loadedTree.LowestInsertedBodyNumber, Is.EqualTo(1), "loaded tree");
-    }
-
-
     private static readonly object[] SourceOfBSearchTestCases =
     {
         new object[] {1L, 0L},
@@ -1362,7 +1321,6 @@ public void Loads_best_known_correctly_on_inserts_followed_by_suggests(long pivo
 
         Assert.That(tree.BestKnownNumber, Is.EqualTo(pivotNumber + 1), "tree");
         Assert.That(tree.LowestInsertedHeader?.Number, Is.EqualTo(1), "loaded tree - lowest header");
-        Assert.That(tree.LowestInsertedBodyNumber, Is.EqualTo(null), "loaded tree - lowest body");
         Assert.That(loadedTree.BestKnownNumber, Is.EqualTo(pivotNumber + 1), "loaded tree");
     }
 

src/Nethermind/Nethermind.Blockchain/BlockTree.Initializer.cs

@@ -16,7 +16,6 @@ public partial class BlockTree
 
     public void RecalculateTreeLevels()
     {
-        LoadLowestInsertedBodyNumber();
         LoadLowestInsertedHeader();
         LoadLowestInsertedBeaconHeader();
         LoadBestKnown();
@@ -115,13 +114,6 @@ private void LoadForkChoiceInfo()
         SafeHash ??= _metadataDb.Get(MetadataDbKeys.SafeBlockHash)?.AsRlpStream().DecodeKeccak();
     }
 
-    private void LoadLowestInsertedBodyNumber()
-    {
-        LowestInsertedBodyNumber =
-            _blockStore.GetMetadata(LowestInsertedBodyNumberDbEntryAddress)?
-                .AsRlpValueContext().DecodeLong();
-    }
-
     private void LoadLowestInsertedBeaconHeader()
     {
         if (_metadataDb.KeyExists(MetadataDbKeys.LowestInsertedBeaconHeaderHash))

src/Nethermind/Nethermind.Blockchain/BlockTree.cs

@@ -34,7 +34,6 @@ namespace Nethermind.Blockchain
     public partial class BlockTree : IBlockTree
     {
         // there is not much logic in the addressing here
-        public static readonly byte[] LowestInsertedBodyNumberDbEntryAddress = ((long)0).ToBigEndianByteArrayWithoutLeadingZeros();
         private static readonly byte[] StateHeadHashDbEntryAddress = new byte[16];
         internal static Hash256 DeletePointerAddressInDb = new(new BitArray(32 * 8, true).ToBytes());
         internal static Hash256 HeadAddressInDb = Keccak.Zero;
@@ -79,22 +78,8 @@ public BlockHeader? LowestInsertedBeaconHeader
 
         private BlockHeader? _lowestInsertedBeaconHeader;
 
-        private long? _lowestInsertedReceiptBlock;
         private long? _highestPersistedState;
 
-        public long? LowestInsertedBodyNumber
-        {
-            get => _lowestInsertedReceiptBlock;
-            set
-            {
-                _lowestInsertedReceiptBlock = value;
-                if (value.HasValue)
-                {
-                    _blockStore.SetMetadata(LowestInsertedBodyNumberDbEntryAddress, Rlp.Encode(value.Value).Bytes);
-                }
-            }
-        }
-
         public long BestKnownNumber { get; private set; }
 
         public long BestKnownBeaconNumber { get; private set; }
@@ -169,7 +154,6 @@ public BlockTree(
                              $"best queued is {BestSuggestedHeader?.Number.ToString() ?? "0"}, " +
                              $"best known is {BestKnownNumber}, " +
                              $"lowest inserted header {LowestInsertedHeader?.Number}, " +
-                             $"body {LowestInsertedBodyNumber}, " +
                              $"lowest sync inserted block number {LowestInsertedBeaconHeader?.Number}");
             ProductInfo.Network = $"{(ChainId == NetworkId ? BlockchainIds.GetBlockchainName(NetworkId) : ChainId)}";
             ThisNodeInfo.AddInfo("Chain ID     :", ProductInfo.Network);

src/Nethermind/Nethermind.Blockchain/BlockTreeOverlay.cs

@@ -32,12 +32,6 @@ public BlockTreeOverlay(IReadOnlyBlockTree baseTree, IBlockTree overlayTree)
     public BlockHeader? BestSuggestedBeaconHeader => _overlayTree.BestSuggestedBeaconHeader ?? _baseTree.BestSuggestedBeaconHeader;
     public BlockHeader? LowestInsertedHeader => _overlayTree.LowestInsertedHeader ?? _baseTree.LowestInsertedHeader;
 
-    public long? LowestInsertedBodyNumber
-    {
-        get => _overlayTree.LowestInsertedBodyNumber ?? _baseTree.LowestInsertedBodyNumber;
-        set => _overlayTree.LowestInsertedBodyNumber = value;
-    }
-
     public BlockHeader? LowestInsertedBeaconHeader
     {
         get => _overlayTree.LowestInsertedBeaconHeader ?? _baseTree.LowestInsertedBeaconHeader;

src/Nethermind/Nethermind.Blockchain/Blocks/IBlockStore.cs

@@ -19,9 +19,5 @@ public interface IBlockStore
     byte[]? GetRlp(long blockNumber, Hash256 blockHash);
     ReceiptRecoveryBlock? GetReceiptRecoveryBlock(long blockNumber, Hash256 blockHash);
     void Cache(Block block);
-
-    // These two are used by blocktree. Try not to use them...
-    void SetMetadata(byte[] key, byte[] value);
-    byte[]? GetMetadata(byte[] key);
     bool HasBlock(long blockNumber, Hash256 blockHash);
 }

src/Nethermind/Nethermind.Blockchain/IBlockTree.cs

@@ -47,11 +47,6 @@ public interface IBlockTree : IBlockFinder
         /// </summary>
         BlockHeader? LowestInsertedHeader { get; }
 
-        /// <summary>
-        /// Lowest body added in reverse fast sync insert
-        /// </summary>
-        long? LowestInsertedBodyNumber { get; set; }
-
         /// <summary>
         /// Lowest header number added in reverse beacon sync insert. Used to determine if BeaconHeaderSync is completed.
         /// </summary>

src/Nethermind/Nethermind.Blockchain/ReadOnlyBlockTree.cs

@@ -32,12 +32,6 @@ public ReadOnlyBlockTree(IBlockTree wrapped)
         public BlockHeader BestSuggestedBeaconHeader => _wrapped.BestSuggestedBeaconHeader;
         public BlockHeader LowestInsertedHeader => _wrapped.LowestInsertedHeader;
 
-        public long? LowestInsertedBodyNumber
-        {
-            get => _wrapped.LowestInsertedBodyNumber;
-            set => _wrapped.LowestInsertedBodyNumber = value;
-        }
-
         public long? BestPersistedState
         {
             get => _wrapped.BestPersistedState;

src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptStorage.cs

@@ -11,7 +11,6 @@ public interface IReceiptStorage : IReceiptFinder
     {
         void Insert(Block block, params TxReceipt[]? txReceipts) => Insert(block, txReceipts, true);
         void Insert(Block block, TxReceipt[]? txReceipts, bool ensureCanonical, WriteFlags writeFlags = WriteFlags.None);
-        long? LowestInsertedReceiptBlockNumber { get; set; }
         long MigratedBlockNumber { get; set; }
         bool HasBlock(long blockNumber, Hash256 hash);
         void EnsureCanonical(Block block);

src/Nethermind/Nethermind.Blockchain/Receipts/InMemoryReceiptStorage.cs

@@ -87,8 +87,6 @@ public void EnsureCanonical(Block block)
             }
         }
 
-        public long? LowestInsertedReceiptBlockNumber { get; set; }
-
         public long MigratedBlockNumber { get; set; }
 
         public int Count => _transactions.Count;

src/Nethermind/Nethermind.Blockchain/Receipts/NullReceiptStorage.cs

@@ -33,12 +33,6 @@ public bool TryGetReceiptsIterator(long blockNumber, Hash256 blockHash, out Rece
             return false;
         }
 
-        public long? LowestInsertedReceiptBlockNumber
-        {
-            get => 0;
-            set { }
-        }
-
         public long MigratedBlockNumber { get; set; } = 0;
 
         public bool HasBlock(long blockNumber, Hash256 hash)

src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs

@@ -25,7 +25,6 @@ public class PersistentReceiptStorage : IReceiptStorage
         private readonly IColumnsDb<ReceiptsColumns> _database;
         private readonly ISpecProvider _specProvider;
         private readonly IReceiptsRecovery _receiptsRecovery;
-        private long? _lowestInsertedReceiptBlock;
         private readonly IDb _blocksDb;
         private readonly IDb _defaultColumn;
         private readonly IDb _transactionDb;
@@ -65,8 +64,6 @@ public PersistentReceiptStorage(
             _storageDecoder = storageDecoder ?? ReceiptArrayStorageDecoder.Instance;
             _receiptConfig = receiptConfig ?? throw new ArgumentNullException(nameof(receiptConfig));
 
-            byte[] lowestBytes = _defaultColumn.Get(Keccak.Zero);
-            _lowestInsertedReceiptBlock = lowestBytes is null ? (long?)null : new RlpStream(lowestBytes).DecodeLong();
             _migratedBlockNumber = Get(MigrationBlockNumberKey, long.MaxValue);
 
             KeyValuePair<byte[], byte[]>? firstValue = _blocksDb.GetAll().FirstOrDefault();
@@ -302,19 +299,6 @@ public void Insert(Block block, TxReceipt[]? txReceipts, bool ensureCanonical =
             }
         }
 
-        public long? LowestInsertedReceiptBlockNumber
-        {
-            get => _lowestInsertedReceiptBlock;
-            set
-            {
-                _lowestInsertedReceiptBlock = value;
-                if (value.HasValue)
-                {
-                    _defaultColumn.Set(Keccak.Zero, Rlp.Encode(value.Value).Bytes);
-                }
-            }
-        }
-
         public long MigratedBlockNumber
         {
             get => _migratedBlockNumber;

src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs

@@ -161,4 +161,7 @@ public interface ISyncConfig : IConfig
 
     [ConfigItem(Description = "_Technical._ Min distance of state sync from best suggested header.", DefaultValue = "32", HiddenFromDocs = true)]
     int StateMinDistanceFromHead { get; set; }
+
+    [ConfigItem(Description = "_Technical._ Run explicit GC after state sync finished.", DefaultValue = "true", HiddenFromDocs = true)]
+    bool GCOnFeedFinished { get; set; }
 }

src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs

@@ -72,6 +72,7 @@ public string? PivotHash
         public bool TrieHealing { get; set; } = true;
         public int StateMaxDistanceFromHead { get; set; } = 128;
         public int StateMinDistanceFromHead { get; set; } = 32;
+        public bool GCOnFeedFinished { get; set; } = true;
 
         public override string ToString()
         {

src/Nethermind/Nethermind.Core.Test/CachedBlockTreeBuilder.cs

@@ -0,0 +1,55 @@
+// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System;
+using Nethermind.Blockchain;
+using Nethermind.Core.Test.Builders;
+using Nethermind.Db;
+using NonBlocking;
+
+namespace Nethermind.Core.Test;
+
+public class CachedBlockTreeBuilder
+{
+    private static ConcurrentDictionary<string, CachedDb> _cachedDbs = new();
+    private record CachedDb(
+        MemDb blocksDb,
+        MemDb metadataDb,
+        MemDb headersDb,
+        MemDb blockNumbersDb,
+        MemDb blockInfo
+    );
+
+    public static IBlockTree BuildCached(string key, Func<BlockTreeBuilder> blockTreeBuilderFactory)
+    {
+        if (_cachedDbs.TryGetValue(key, out CachedDb? db))
+        {
+            return Build.A.BlockTree()
+                .WithBlocksDb(MemDb.CopyFrom(db.blocksDb))
+                .WithMetadataDb(MemDb.CopyFrom(db.metadataDb))
+                .WithHeadersDb(MemDb.CopyFrom(db.headersDb))
+                .WithBlocksNumberDb(MemDb.CopyFrom(db.blockNumbersDb))
+                .WithBlockInfoDb(MemDb.CopyFrom(db.blockInfo))
+                .TestObject;
+        }
+        else
+        {
+            BlockTreeBuilder builder = blockTreeBuilderFactory();
+            CachedDb cachedValue = new CachedDb(
+                MemDb.CopyFrom(builder.BlocksDb),
+                MemDb.CopyFrom(builder.MetadataDb),
+                MemDb.CopyFrom(builder.HeadersDb),
+                MemDb.CopyFrom(builder.BlockNumbersDb),
+                MemDb.CopyFrom(builder.BlockInfoDb)
+            );
+            _cachedDbs.GetOrAdd(key, cachedValue);
+
+            return builder.TestObject;
+        }
+    }
+
+    public static IBlockTree OfLength(int length)
+    {
+        return BuildCached($"{nameof(CachedBlockTreeBuilder)}-{length}", () => Build.A.BlockTree().OfChainLength(length));
+    }
+}

src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs

@@ -153,6 +153,15 @@ public static ContainerBuilder Map<TFrom, TTo>(this ContainerBuilder builder, Fu
 
         return builder;
     }
+
+    public static ContainerBuilder Bind<TFrom, TTo>(this ContainerBuilder builder) where TFrom : TTo where TTo : notnull
+    {
+        builder.Register((it) => it.Resolve<TFrom>())
+            .As<TTo>()
+            .ExternallyOwned();
+
+        return builder;
+    }
 }
 
 /// <summary>

src/Nethermind/Nethermind.Core/Extensions/Bytes.cs

@@ -27,8 +27,12 @@ public static unsafe partial class Bytes
         public static readonly IEqualityComparer<byte[]> EqualityComparer = new BytesEqualityComparer();
         public static readonly IEqualityComparer<byte[]?> NullableEqualityComparer = new NullableBytesEqualityComparer();
         public static readonly BytesComparer Comparer = new();
-        public static readonly ReadOnlyMemory<byte> ZeroByte = new byte[] { 0 };
-        public static readonly ReadOnlyMemory<byte> OneByte = new byte[] { 1 };
+        // The ReadOnlyMemory<byte> needs to be initialized = or it will be created each time.
+        public static ReadOnlyMemory<byte> ZeroByte = new byte[] { 0 };
+        public static ReadOnlyMemory<byte> OneByte = new byte[] { 1 };
+        // The Jit converts a ReadOnlySpan<byte> => new byte[] to a data section load, no allocation.
+        public static ReadOnlySpan<byte> ZeroByteSpan => new byte[] { 0 };
+        public static ReadOnlySpan<byte> OneByteSpan => new byte[] { 1 };
 
         public const string ZeroHexValue = "0x0";
         public const string ZeroValue = "0";
@@ -210,7 +214,7 @@ public static ReadOnlySpan<byte> WithoutLeadingZeros(this Span<byte> bytes)
 
         public static ReadOnlySpan<byte> WithoutLeadingZeros(this ReadOnlySpan<byte> bytes)
         {
-            if (bytes.Length == 0) return ZeroByte.Span;
+            if (bytes.Length == 0) return ZeroByteSpan;
 
             int nonZeroIndex = bytes.IndexOfAnyExcept((byte)0);
             // Keep one or it will be interpreted as null

src/Nethermind/Nethermind.Db/MemDb.cs

@@ -27,6 +27,17 @@ public MemDb(string name)
             Name = name;
         }
 
+        public static MemDb CopyFrom(IDb anotherDb)
+        {
+            MemDb newDb = new MemDb();
+            foreach (KeyValuePair<byte[], byte[]> kv in anotherDb.GetAll())
+            {
+                newDb[kv.Key] = kv.Value;
+            }
+
+            return newDb;
+        }
+
         public MemDb() : this(0, 0)
         {
         }