Merge branch 'master' into 7748_additional_rpc_url_table

src/Nethermind/Nethermind.Api/IApiWithNetwork.cs

@@ -47,6 +47,7 @@ public interface IApiWithNetwork : IApiWithBlockchain
         ISynchronizer? Synchronizer { get; }
         ISyncModeSelector SyncModeSelector { get; }
         ISyncProgressResolver? SyncProgressResolver { get; }
+        ISyncPointers? SyncPointers { get; }
         IPivot? Pivot { get; set; }
         ISyncPeerPool? SyncPeerPool { get; }
         IPeerDifficultyRefreshPool? PeerDifficultyRefreshPool { get; }

src/Nethermind/Nethermind.Api/IBasicApi.cs

@@ -7,6 +7,7 @@
 using Autofac;
 using Nethermind.Abi;
 using Nethermind.Api.Extensions;
+using Nethermind.Blockchain.Receipts;
 using Nethermind.Blockchain.Synchronization;
 using Nethermind.Config;
 using Nethermind.Core;
@@ -63,6 +64,7 @@ public ContainerBuilder ConfigureContainerBuilderFromBasicApi(ContainerBuilder b
             builder
                 .AddPropertiesFrom<IBasicApi>(this)
                 .AddSingleton(ConfigProvider.GetConfig<ISyncConfig>())
+                .AddSingleton(ConfigProvider.GetConfig<IReceiptConfig>())
                 .AddModule(new DbModule());
 
             return builder;

src/Nethermind/Nethermind.Api/NethermindApi.cs

@@ -190,6 +190,7 @@ public ISealEngine SealEngine
         public ISyncModeSelector SyncModeSelector => ApiWithNetworkServiceContainer?.Resolve<ISyncModeSelector>()!;
 
         public ISyncProgressResolver? SyncProgressResolver => ApiWithNetworkServiceContainer?.Resolve<ISyncProgressResolver>();
+        public ISyncPointers? SyncPointers => ApiWithNetworkServiceContainer?.Resolve<ISyncPointers>();
         public IBetterPeerStrategy? BetterPeerStrategy { get; set; }
         public IPivot? Pivot { get; set; }
         public ISyncPeerPool? SyncPeerPool => ApiWithNetworkServiceContainer?.Resolve<ISyncPeerPool>();

src/Nethermind/Nethermind.Blockchain.Test/BlockTreeTests.cs

@@ -1170,47 +1170,38 @@ public void Loads_lowest_inserted_header_correctly(long beginIndex, long inserte
         Assert.That(loadedTree.LowestInsertedHeader?.Number, Is.EqualTo(expectedResult), "loaded tree");
     }
 
-    [Test, MaxTime(Timeout.MaxTestTime), TestCaseSource(nameof(SourceOfBSearchTestCases))]
-    public void Loads_lowest_inserted_body_correctly(long beginIndex, long insertedBlocks)
+    [TestCase(5, 10)]
+    [TestCase(10, 10)]
+    [TestCase(12, 0)]
+    public void Does_not_load_bestKnownNumber_before_syncPivot(long syncPivot, long expectedBestKnownNumber)
     {
-        // left old code to prove that it does not matter for the result nowadays
-        // we store and no longer binary search lowest body number
-
-        MemDb blocksDb = new();
-        blocksDb.Set(BlockTree.LowestInsertedBodyNumberDbEntryAddress, Rlp.Encode(1L).Bytes);
-
         SyncConfig syncConfig = new()
         {
-            PivotNumber = beginIndex.ToString(),
+            FastSync = true,
+            PivotNumber = $"{syncPivot}"
         };
 
-        BlockTreeBuilder builder = Build.A.BlockTree()
-            .WithBlocksDb(blocksDb)
-            .WithSyncConfig(syncConfig)
-            .WithoutSettingHead;
-
-        BlockTree tree = builder.TestObject;
-
-        tree.SuggestBlock(Build.A.Block.Genesis.TestObject);
+        MemDb blockInfosDb = new MemDb();
+        MemDb headersDb = new MemDb();
+        MemDb blockDb = new MemDb();
 
-        for (long i = beginIndex; i > beginIndex - insertedBlocks; i--)
-        {
-            Block block = Build.A.Block.WithNumber(i).WithTotalDifficulty(i).TestObject;
-            tree.Insert(block.Header);
-            tree.Insert(block);
-        }
+        _ = Build.A.BlockTree()
+            .WithHeadersDb(headersDb)
+            .WithBlockInfoDb(blockInfosDb)
+            .WithBlocksDb(blockDb)
+            .OfChainLength(11)
+            .TestObject;
 
-        BlockTree loadedTree = Build.A.BlockTree()
-            .WithoutSettingHead
-            .WithDatabaseFrom(builder)
+        BlockTree tree = Build.A.BlockTree()
             .WithSyncConfig(syncConfig)
+            .WithHeadersDb(headersDb)
+            .WithBlockInfoDb(blockInfosDb)
+            .WithBlocksDb(blockDb)
             .TestObject;
 
-        Assert.That(tree.LowestInsertedBodyNumber, Is.EqualTo(null), "tree");
-        Assert.That(loadedTree.LowestInsertedBodyNumber, Is.EqualTo(1), "loaded tree");
+        Assert.That(tree.BestKnownNumber, Is.EqualTo(expectedBestKnownNumber));
     }
 
-
     private static readonly object[] SourceOfBSearchTestCases =
     {
         new object[] {1L, 0L},
@@ -1362,7 +1353,6 @@ public void Loads_best_known_correctly_on_inserts_followed_by_suggests(long pivo
 
         Assert.That(tree.BestKnownNumber, Is.EqualTo(pivotNumber + 1), "tree");
         Assert.That(tree.LowestInsertedHeader?.Number, Is.EqualTo(1), "loaded tree - lowest header");
-        Assert.That(tree.LowestInsertedBodyNumber, Is.EqualTo(null), "loaded tree - lowest body");
         Assert.That(loadedTree.BestKnownNumber, Is.EqualTo(pivotNumber + 1), "loaded tree");
     }
 

src/Nethermind/Nethermind.Blockchain/BlockTree.Initializer.cs

@@ -16,7 +16,6 @@ public partial class BlockTree
 
     public void RecalculateTreeLevels()
     {
-        LoadLowestInsertedBodyNumber();
         LoadLowestInsertedHeader();
         LoadLowestInsertedBeaconHeader();
         LoadBestKnown();
@@ -115,13 +114,6 @@ private void LoadForkChoiceInfo()
         SafeHash ??= _metadataDb.Get(MetadataDbKeys.SafeBlockHash)?.AsRlpStream().DecodeKeccak();
     }
 
-    private void LoadLowestInsertedBodyNumber()
-    {
-        LowestInsertedBodyNumber =
-            _blockStore.GetMetadata(LowestInsertedBodyNumberDbEntryAddress)?
-                .AsRlpValueContext().DecodeLong();
-    }
-
     private void LoadLowestInsertedBeaconHeader()
     {
         if (_metadataDb.KeyExists(MetadataDbKeys.LowestInsertedBeaconHeaderHash))
@@ -148,30 +140,15 @@ private void LoadBestKnown()
 
         long right = Math.Max(0, left) + BestKnownSearchLimit;
 
-        long bestKnownNumberFound =
-            BinarySearchBlockNumber(1, left, LevelExists) ?? 0;
-        long bestKnownNumberAlternative =
-            BinarySearchBlockNumber(left, right, LevelExists) ?? 0;
-
-        long bestSuggestedHeaderNumber =
-            BinarySearchBlockNumber(1, left, HeaderExists) ?? 0;
-        long bestSuggestedHeaderNumberAlternative
-            = BinarySearchBlockNumber(left, right, HeaderExists) ?? 0;
-
-        long bestSuggestedBodyNumber
-            = BinarySearchBlockNumber(1, left, BodyExists) ?? 0;
-        long bestSuggestedBodyNumberAlternative
-            = BinarySearchBlockNumber(left, right, BodyExists) ?? 0;
+        long bestKnownNumberFound = BinarySearchBlockNumber(left, right, LevelExists) ?? 0;
+        long bestSuggestedHeaderNumber = BinarySearchBlockNumber(left, right, HeaderExists) ?? 0;
+        long bestSuggestedBodyNumber = BinarySearchBlockNumber(left, right, BodyExists) ?? 0;
 
         if (_logger.IsInfo)
             _logger.Info("Numbers resolved, " +
-                         $"level = Max({bestKnownNumberFound}, {bestKnownNumberAlternative}), " +
-                         $"header = Max({bestSuggestedHeaderNumber}, {bestSuggestedHeaderNumberAlternative}), " +
-                         $"body = Max({bestSuggestedBodyNumber}, {bestSuggestedBodyNumberAlternative})");
-
-        bestKnownNumberFound = Math.Max(bestKnownNumberFound, bestKnownNumberAlternative);
-        bestSuggestedHeaderNumber = Math.Max(bestSuggestedHeaderNumber, bestSuggestedHeaderNumberAlternative);
-        bestSuggestedBodyNumber = Math.Max(bestSuggestedBodyNumber, bestSuggestedBodyNumberAlternative);
+                         $"level = {bestKnownNumberFound}, " +
+                         $"header = {bestSuggestedHeaderNumber}, " +
+                         $"body = {bestSuggestedBodyNumber}");
 
         if (bestKnownNumberFound < 0 ||
             bestSuggestedHeaderNumber < 0 ||
@@ -195,7 +172,7 @@ long bestSuggestedBodyNumberAlternative
             }
         }
 
-        BestKnownNumber = Math.Max(bestKnownNumberFound, bestKnownNumberAlternative);
+        BestKnownNumber = bestKnownNumberFound;
         BestSuggestedHeader = FindHeader(bestSuggestedHeaderNumber, BlockTreeLookupOptions.None);
         BlockHeader? bestSuggestedBodyHeader = FindHeader(bestSuggestedBodyNumber, BlockTreeLookupOptions.None);
         BestSuggestedBody = bestSuggestedBodyHeader is null

src/Nethermind/Nethermind.Blockchain/BlockTree.cs

@@ -34,7 +34,6 @@ namespace Nethermind.Blockchain
     public partial class BlockTree : IBlockTree
     {
         // there is not much logic in the addressing here
-        public static readonly byte[] LowestInsertedBodyNumberDbEntryAddress = ((long)0).ToBigEndianByteArrayWithoutLeadingZeros();
         private static readonly byte[] StateHeadHashDbEntryAddress = new byte[16];
         internal static Hash256 DeletePointerAddressInDb = new(new BitArray(32 * 8, true).ToBytes());
         internal static Hash256 HeadAddressInDb = Keccak.Zero;
@@ -79,22 +78,8 @@ public BlockHeader? LowestInsertedBeaconHeader
 
         private BlockHeader? _lowestInsertedBeaconHeader;
 
-        private long? _lowestInsertedReceiptBlock;
         private long? _highestPersistedState;
 
-        public long? LowestInsertedBodyNumber
-        {
-            get => _lowestInsertedReceiptBlock;
-            set
-            {
-                _lowestInsertedReceiptBlock = value;
-                if (value.HasValue)
-                {
-                    _blockStore.SetMetadata(LowestInsertedBodyNumberDbEntryAddress, Rlp.Encode(value.Value).Bytes);
-                }
-            }
-        }
-
         public long BestKnownNumber { get; private set; }
 
         public long BestKnownBeaconNumber { get; private set; }
@@ -169,7 +154,6 @@ public BlockTree(
                              $"best queued is {BestSuggestedHeader?.Number.ToString() ?? "0"}, " +
                              $"best known is {BestKnownNumber}, " +
                              $"lowest inserted header {LowestInsertedHeader?.Number}, " +
-                             $"body {LowestInsertedBodyNumber}, " +
                              $"lowest sync inserted block number {LowestInsertedBeaconHeader?.Number}");
             ProductInfo.Network = $"{(ChainId == NetworkId ? BlockchainIds.GetBlockchainName(NetworkId) : ChainId)}";
             ThisNodeInfo.AddInfo("Chain ID     :", ProductInfo.Network);

src/Nethermind/Nethermind.Blockchain/BlockTreeExtensions.cs

@@ -10,5 +10,14 @@ public static class BlockTreeExtensions
         public static ReadOnlyBlockTree AsReadOnly(this IBlockTree blockTree) => new(blockTree);
 
         public static BlockHeader? GetProducedBlockParent(this IBlockTree blockTree, BlockHeader? parentHeader) => parentHeader ?? blockTree.Head?.Header;
+
+        public static (bool isSyncing, long headNumber, long bestSuggested) IsSyncing(this IBlockTree blockTree, int maxDistanceForSynced = 0)
+        {
+            long bestSuggestedNumber = blockTree.FindBestSuggestedHeader()?.Number ?? 0;
+            long headNumberOrZero = blockTree.Head?.Number ?? 0;
+            bool isSyncing = bestSuggestedNumber == 0 || bestSuggestedNumber > headNumberOrZero + maxDistanceForSynced;
+
+            return (isSyncing, headNumberOrZero, bestSuggestedNumber);
+        }
     }
 }

src/Nethermind/Nethermind.Blockchain/BlockTreeOverlay.cs

@@ -32,12 +32,6 @@ public BlockTreeOverlay(IReadOnlyBlockTree baseTree, IBlockTree overlayTree)
     public BlockHeader? BestSuggestedBeaconHeader => _overlayTree.BestSuggestedBeaconHeader ?? _baseTree.BestSuggestedBeaconHeader;
     public BlockHeader? LowestInsertedHeader => _overlayTree.LowestInsertedHeader ?? _baseTree.LowestInsertedHeader;
 
-    public long? LowestInsertedBodyNumber
-    {
-        get => _overlayTree.LowestInsertedBodyNumber ?? _baseTree.LowestInsertedBodyNumber;
-        set => _overlayTree.LowestInsertedBodyNumber = value;
-    }
-
     public BlockHeader? LowestInsertedBeaconHeader
     {
         get => _overlayTree.LowestInsertedBeaconHeader ?? _baseTree.LowestInsertedBeaconHeader;

src/Nethermind/Nethermind.Blockchain/Blocks/IBlockStore.cs

@@ -19,9 +19,5 @@ public interface IBlockStore
     byte[]? GetRlp(long blockNumber, Hash256 blockHash);
     ReceiptRecoveryBlock? GetReceiptRecoveryBlock(long blockNumber, Hash256 blockHash);
     void Cache(Block block);
-
-    // These two are used by blocktree. Try not to use them...
-    void SetMetadata(byte[] key, byte[] value);
-    byte[]? GetMetadata(byte[] key);
     bool HasBlock(long blockNumber, Hash256 blockHash);
 }

src/Nethermind/Nethermind.Blockchain/ChainHeadInfoProvider.cs

@@ -17,6 +17,10 @@ namespace Nethermind.Blockchain
 {
     public class ChainHeadInfoProvider : IChainHeadInfoProvider
     {
+        private readonly IBlockTree _blockTree;
+        // For testing
+        public bool HasSynced { private get; init; }
+
         public ChainHeadInfoProvider(ISpecProvider specProvider, IBlockTree blockTree, IStateReader stateReader, ICodeInfoRepository codeInfoRepository)
             : this(new ChainHeadSpecProvider(specProvider, blockTree), blockTree, new ChainHeadReadOnlyStateProvider(blockTree, stateReader), codeInfoRepository)
         {
@@ -35,6 +39,7 @@ public ChainHeadInfoProvider(IChainHeadSpecProvider specProvider, IBlockTree blo
             CodeInfoRepository = codeInfoRepository;
 
             blockTree.BlockAddedToMain += OnHeadChanged;
+            _blockTree = blockTree;
         }
 
         public IChainHeadSpecProvider SpecProvider { get; }
@@ -51,6 +56,20 @@ public ChainHeadInfoProvider(IChainHeadSpecProvider specProvider, IBlockTree blo
 
         public UInt256 CurrentFeePerBlobGas { get; internal set; }
 
+        public bool IsSyncing
+        {
+            get
+            {
+                if (HasSynced)
+                {
+                    return false;
+                }
+
+                (bool isSyncing, _, _) = _blockTree.IsSyncing(maxDistanceForSynced: 2);
+                return isSyncing;
+            }
+        }
+
         public event EventHandler<BlockReplacementEventArgs>? HeadChanged;
 
         private void OnHeadChanged(object? sender, BlockReplacementEventArgs e)

src/Nethermind/Nethermind.Blockchain/IBlockTree.cs

@@ -47,11 +47,6 @@ public interface IBlockTree : IBlockFinder
         /// </summary>
         BlockHeader? LowestInsertedHeader { get; }
 
-        /// <summary>
-        /// Lowest body added in reverse fast sync insert
-        /// </summary>
-        long? LowestInsertedBodyNumber { get; set; }
-
         /// <summary>
         /// Lowest header number added in reverse beacon sync insert. Used to determine if BeaconHeaderSync is completed.
         /// </summary>

src/Nethermind/Nethermind.Blockchain/ReadOnlyBlockTree.cs

@@ -32,12 +32,6 @@ public ReadOnlyBlockTree(IBlockTree wrapped)
         public BlockHeader BestSuggestedBeaconHeader => _wrapped.BestSuggestedBeaconHeader;
         public BlockHeader LowestInsertedHeader => _wrapped.LowestInsertedHeader;
 
-        public long? LowestInsertedBodyNumber
-        {
-            get => _wrapped.LowestInsertedBodyNumber;
-            set => _wrapped.LowestInsertedBodyNumber = value;
-        }
-
         public long? BestPersistedState
         {
             get => _wrapped.BestPersistedState;

src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptStorage.cs

@@ -11,7 +11,6 @@ public interface IReceiptStorage : IReceiptFinder
     {
         void Insert(Block block, params TxReceipt[]? txReceipts) => Insert(block, txReceipts, true);
         void Insert(Block block, TxReceipt[]? txReceipts, bool ensureCanonical, WriteFlags writeFlags = WriteFlags.None);
-        long? LowestInsertedReceiptBlockNumber { get; set; }
         long MigratedBlockNumber { get; set; }
         bool HasBlock(long blockNumber, Hash256 hash);
         void EnsureCanonical(Block block);

src/Nethermind/Nethermind.Blockchain/Receipts/InMemoryReceiptStorage.cs

@@ -87,8 +87,6 @@ public void EnsureCanonical(Block block)
             }
         }
 
-        public long? LowestInsertedReceiptBlockNumber { get; set; }
-
         public long MigratedBlockNumber { get; set; }
 
         public int Count => _transactions.Count;

src/Nethermind/Nethermind.Blockchain/Receipts/NullReceiptStorage.cs

@@ -33,12 +33,6 @@ public bool TryGetReceiptsIterator(long blockNumber, Hash256 blockHash, out Rece
             return false;
         }
 
-        public long? LowestInsertedReceiptBlockNumber
-        {
-            get => 0;
-            set { }
-        }
-
         public long MigratedBlockNumber { get; set; } = 0;
 
         public bool HasBlock(long blockNumber, Hash256 hash)

src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs

@@ -25,7 +25,6 @@ public class PersistentReceiptStorage : IReceiptStorage
         private readonly IColumnsDb<ReceiptsColumns> _database;
         private readonly ISpecProvider _specProvider;
         private readonly IReceiptsRecovery _receiptsRecovery;
-        private long? _lowestInsertedReceiptBlock;
         private readonly IDb _blocksDb;
         private readonly IDb _defaultColumn;
         private readonly IDb _transactionDb;
@@ -65,8 +64,6 @@ public PersistentReceiptStorage(
             _storageDecoder = storageDecoder ?? ReceiptArrayStorageDecoder.Instance;
             _receiptConfig = receiptConfig ?? throw new ArgumentNullException(nameof(receiptConfig));
 
-            byte[] lowestBytes = _defaultColumn.Get(Keccak.Zero);
-            _lowestInsertedReceiptBlock = lowestBytes is null ? (long?)null : new RlpStream(lowestBytes).DecodeLong();
             _migratedBlockNumber = Get(MigrationBlockNumberKey, long.MaxValue);
 
             KeyValuePair<byte[], byte[]>? firstValue = _blocksDb.GetAll().FirstOrDefault();
@@ -302,19 +299,6 @@ public void Insert(Block block, TxReceipt[]? txReceipts, bool ensureCanonical =
             }
         }
 
-        public long? LowestInsertedReceiptBlockNumber
-        {
-            get => _lowestInsertedReceiptBlock;
-            set
-            {
-                _lowestInsertedReceiptBlock = value;
-                if (value.HasValue)
-                {
-                    _defaultColumn.Set(Keccak.Zero, Rlp.Encode(value.Value).Bytes);
-                }
-            }
-        }
-
         public long MigratedBlockNumber
         {
             get => _migratedBlockNumber;

src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs

@@ -161,4 +161,7 @@ public interface ISyncConfig : IConfig
 
     [ConfigItem(Description = "_Technical._ Min distance of state sync from best suggested header.", DefaultValue = "32", HiddenFromDocs = true)]
     int StateMinDistanceFromHead { get; set; }
+
+    [ConfigItem(Description = "_Technical._ Run explicit GC after state sync finished.", DefaultValue = "true", HiddenFromDocs = true)]
+    bool GCOnFeedFinished { get; set; }
 }