Merge branch 'master' into trie-store-channels

.editorconfig

@@ -92,6 +92,6 @@ csharp_style_var_elsewhere = false:suggestion
 
 # Coding convention
 
-dotnet_diagnostic.IDE0005.severity = error
+dotnet_diagnostic.IDE0005.severity = suggestion
 dotnet_diagnostic.IDE0041.severity = error
 

.github/workflows/codeql.yml

@@ -20,6 +20,11 @@ jobs:
       matrix:
         language: ['csharp']
     steps:
+      - name: Free up disk space
+        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be #v1.3.1
+        with:
+          large-packages: false
+          tool-cache: false
       - name: Check out repository
         uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 #v4.2.0
         with:

.github/workflows/publish-packages.yml

@@ -60,7 +60,7 @@ jobs:
     needs: publish-ppa
     steps:
       - name: Wait for PPA package to be published
-        run: sleep 1.5h
+        run: sleep 2h
       - name: Install PPA dependencies
         run: |
           sudo apt-get update

.github/workflows/sync-supported-chains.yml

@@ -3,9 +3,13 @@ name: Sync Supported Chains
 on:
   workflow_run:
     workflows: ["Publish Docker image"]
-    branches: [master, release/*]
+    branches: [release/*]
     types:
       - completed
+
+  schedule:
+    - cron: "0 0 * * *"
+
   workflow_dispatch:
     inputs:
       nethermind_image:
@@ -84,15 +88,31 @@ jobs:
     timeout-minutes: ${{ matrix.config.timeout }}
 
     steps:
+      - name: Authenticate App
+        id: gh-app
+        uses: actions/create-github-app-token@v1
+        with:
+          app-id: ${{ vars.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: "nethermind,post-merge-smoke-tests"
+
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
           clean: true
+
+      - name: Checkout tests repository
+        uses: actions/checkout@v4
+        with:
+          repository: NethermindEth/post-merge-smoke-tests
+          path: tests
+          token: ${{ steps.gh-app.outputs.token }}
+          clean: true
 
       - name: Installing requirements
         run: |
           sudo apt-get update
-          sudo apt-get install -y make build-essential jq screen lshw dmidecode fio
+          sudo apt-get install -y make build-essential jq screen lshw dmidecode fio zip
 
       - name: Setup Go environment
         uses: actions/setup-go@v5
@@ -154,6 +174,9 @@ jobs:
 
             stripped_network="${network#base-}"
             stripped_network="${stripped_network#op-}"
+
+            mkdir -p execution-data-op/logs/configs
+            mv ../tests/predefined_configs/customNLog.config execution-data-op/logs/configs/customNLog.config
             
             ./build/sedge generate \
             --logging none \
@@ -165,6 +188,7 @@ jobs:
             --network $stripped_network \
             --consensus-url $CONSENSUS_URL \
             --execution-api-url $EXECUTION_URL \
+            --el-op-extra-flag loggerConfigSource=/nethermind/data/logs/configs/customNLog.config \
             --el-op-extra-flag Sync.VerifyTrieOnStateSyncFinished=true \
             $extra_param
           elif [[ "$network" == taiko-* ]]; then
@@ -180,6 +204,9 @@ jobs:
               echo "Unknown network"
               exit 1
             fi
+
+            mkdir -p execution-data-taiko/logs/configs
+            mv ../tests/predefined_configs/customNLog.config execution-data-taiko/logs/configs/customNLog.config
             
             ./build/sedge generate \
             --logging none \
@@ -191,9 +218,13 @@ jobs:
             --network $stripped_network \
             --consensus-url $CONSENSUS_URL \
             --execution-api-url $EXECUTION_URL \
+            --el-l2-extra-flag loggerConfigSource=/nethermind/data/logs/configs/customNLog.config \
             --el-l2-extra-flag Sync.VerifyTrieOnStateSyncFinished=true \
             $extra_param
           else
+            mkdir -p execution-data/logs/configs
+            mv ../tests/predefined_configs/customNLog.config execution-data/logs/configs/customNLog.config
+          
             ./build/sedge generate \
             --logging none \
             -p $GITHUB_WORKSPACE/sedge \
@@ -209,6 +240,7 @@ jobs:
             --el-extra-flag Sync.DownloadReceiptsInFastSync=false \
             --el-extra-flag JsonRpc.EnabledModules=[Eth,Subscribe,Trace,TxPool,Web3,Personal,Proof,Net,Parity,Health,Rpc,Debug] \
             --el-extra-flag Sync.VerifyTrieOnStateSyncFinished=true \
+            --el-extra-flag loggerConfigSource=/nethermind/data/logs/configs/customNLog.config \
             --el-extra-flag Sync.SnapSync=true \
             --checkpoint-sync-url=${{ matrix.config.checkpoint-sync-url }}
           fi          
@@ -222,6 +254,30 @@ jobs:
           NETWORK: ${{ matrix.config.network }}
         run: |
           python scripts/waitForSync.py
+
+      - name: Get Nethermind Debug Logs
+        if: failure()
+        id: find-and-upload
+        run: |      
+          target_dir=$(find . -type d -name "*execution-data*" | head -n 1)
+          cd "$target_dir"
+          cd logs
+          ls
+          log_file=$(find . -type f -name "*logs.txt" | head -n 1)
+          echo "Found log file: $log_file"
+          artifact_name=$(basename "$log_file" .txt)-${{ github.run_id }}
+          zip "${artifact_name}.zip" "$log_file"
+          echo "Compressed log file into ${artifact_name}.zip"
+          echo "artifact_name=$artifact_name" >> $GITHUB_ENV
+          echo "artifact_path=$(pwd)/${artifact_name}.zip" >> $GITHUB_ENV
+
+      - name: Upload Nethermind Debug logs artifact
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.artifact_name }}
+          path: ${{ env.artifact_path }}
+          retention-days: 7
 
       - name: Get Consensus Logs
         if: always() && matrix.config.network != 'joc-mainnet' && matrix.config.network != 'joc-testnet' && matrix.config.network != 'linea-mainnet' && matrix.config.network != 'linea-sepolia'

README.md

@@ -22,7 +22,7 @@ Nethermind documentation is available at [docs.nethermind.io](https://docs.nethe
 
 ### Supported networks
 
-**Ethereum** · **Gnosis** · **Optimism** · **Base** · **Taiko** · **Energy Web**
+**Ethereum** · **Gnosis** · **Optimism** · **Base** · **Taiko** · **Linea** · **Energy Web**
 
 ## Download and run
 

scripts/workflow_config/sync_testnets_matrix.json

@@ -37,47 +37,47 @@
     "cl_image": "chainsafe/lodestar:latest",
     "checkpoint-sync-url": "https://mainnet-checkpoint-sync.attestant.io/",
     "timeout": 180,
-    "agent": "g6-standard-16"
+    "agent": "g6-standard-6"
   },
   {
     "network": "op-sepolia",
     "cl": "",
     "cl_image": "",
     "checkpoint-sync-url": "",
     "timeout": 180,
-    "agent": "g6-standard-8"
+    "agent": "g6-standard-6"
   },
   {
     "network": "base-sepolia",
     "cl": "",
     "cl_image": "",
     "checkpoint-sync-url": "",
     "timeout": 180,
-    "agent": "g6-standard-8"
+    "agent": "g6-standard-6"
   },
   {
     "network": "op-mainnet",
     "cl": "",
     "cl_image": "",
     "checkpoint-sync-url": "",
     "timeout": 600,
-    "agent": "g6-standard-16"
+    "agent": "g6-standard-6"
   },
   {
     "network": "base-mainnet",
     "cl": "",
     "cl_image": "",
     "checkpoint-sync-url": "",
     "timeout": 600,
-    "agent": "g6-standard-16"
+    "agent": "g6-standard-6"
   },
   {
     "network": "taiko-mainnet",
     "cl": "",
     "cl_image": "",
     "checkpoint-sync-url": "",
     "timeout": 600,
-    "agent": "g6-standard-8"
+    "agent": "g6-standard-6"
   },
   {
     "network": "taiko-hekla",

src/Nethermind/Directory.Build.props

@@ -3,7 +3,7 @@
   <PropertyGroup>
     <Configuration Condition="'$(Configuration)' == ''">Debug</Configuration>
     <ContinuousIntegrationBuild Condition="'$(CI)' == 'true'">true</ContinuousIntegrationBuild>
-    <LangVersion>latest</LangVersion>
+    <LangVersion>13.0</LangVersion>
     <NuGetAuditMode>direct</NuGetAuditMode>
     <TargetFramework>net9.0</TargetFramework>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
@@ -15,7 +15,7 @@
     <Copyright>Demerzel Solutions Limited</Copyright>
     <Product>Nethermind</Product>
     <SourceRevisionId Condition="'$(Commit)' != ''">$(Commit)</SourceRevisionId>
-    <VersionPrefix>1.30.0</VersionPrefix>
+    <VersionPrefix>1.31.0</VersionPrefix>
     <VersionSuffix>unstable</VersionSuffix>
   </PropertyGroup>
 

src/Nethermind/Ethereum.Test.Base/JsonToEthereumTest.cs

@@ -323,7 +323,6 @@ public static IEnumerable<GeneralStateTest> Convert(string json)
             List<GeneralStateTest> tests = new();
             foreach (KeyValuePair<string, GeneralStateTestJson> namedTest in testsInFile)
             {
-                Console.WriteLine($"Loading {namedTest.Key}\n {namedTest.Value.Post}");
                 tests.AddRange(Convert(namedTest.Key, namedTest.Value));
             }
 

src/Nethermind/Ethereum.Trie.Test/TrieTests.cs

@@ -24,7 +24,6 @@ public class TrieTests
         [SetUp]
         public void Setup()
         {
-            TrieNode.AllowBranchValues = true;
             _db = new MemDb();
         }
 
@@ -59,7 +58,9 @@ private static IEnumerable<TrieTest> LoadTests()
         {
             return TestLoader.LoadFromFile<Dictionary<string, TrieTestArraysJson>, TrieTest>(
                 "trietest.json",
-                dwj => dwj.Select(Convert));
+                dwj => dwj.Select(Convert))
+                // Remove branch value tests
+                .Where(t => t.Input.All(kvp => kvp.Key.Length == 32));
         }
 
         private static IEnumerable<TrieTest> LoadSecureTests()
@@ -73,15 +74,19 @@ private static IEnumerable<TrieTest> LoadAnyOrderTests()
         {
             IEnumerable<TrieTest> tests = TestLoader.LoadFromFile<Dictionary<string, TrieTestJson>, TrieTest>(
                 "trieanyorder.json",
-                dwj => dwj.Select(p => new TrieTest(p.Key, p.Value.In.ToList(), p.Value.Root)));
+                dwj => dwj.Select(p => new TrieTest(p.Key, p.Value.In.ToList(), p.Value.Root)))
+                    // Remove branch value tests
+                    .Where(t => t.Input.All(kvp => kvp.Key.Length == 32));
             return GetTestPermutations(tests);
         }
 
         private static IEnumerable<TrieTest> LoadHexEncodedSecureTests()
         {
             IEnumerable<TrieTest> tests = TestLoader.LoadFromFile<Dictionary<string, TrieTestJson>, TrieTest>(
                 "hex_encoded_securetrie_test.json",
-                dwj => dwj.Select(p => new TrieTest(p.Key, p.Value.In.ToList(), p.Value.Root)));
+                dwj => dwj.Select(p => new TrieTest(p.Key, p.Value.In.ToList(), p.Value.Root)))
+                    // Remove branch value tests
+                    .Where(t => t.Input.All(kvp => kvp.Key.Length == 32));
             return GetTestPermutations(tests);
         }
 

src/Nethermind/Nethermind.Api/IApiWithNetwork.cs

@@ -47,6 +47,7 @@ public interface IApiWithNetwork : IApiWithBlockchain
         ISynchronizer? Synchronizer { get; }
         ISyncModeSelector SyncModeSelector { get; }
         ISyncProgressResolver? SyncProgressResolver { get; }
+        ISyncPointers? SyncPointers { get; }
         IPivot? Pivot { get; set; }
         ISyncPeerPool? SyncPeerPool { get; }
         IPeerDifficultyRefreshPool? PeerDifficultyRefreshPool { get; }

src/Nethermind/Nethermind.Api/IBasicApi.cs

@@ -7,6 +7,7 @@
 using Autofac;
 using Nethermind.Abi;
 using Nethermind.Api.Extensions;
+using Nethermind.Blockchain.Receipts;
 using Nethermind.Blockchain.Synchronization;
 using Nethermind.Config;
 using Nethermind.Core;
@@ -63,6 +64,7 @@ public ContainerBuilder ConfigureContainerBuilderFromBasicApi(ContainerBuilder b
             builder
                 .AddPropertiesFrom<IBasicApi>(this)
                 .AddSingleton(ConfigProvider.GetConfig<ISyncConfig>())
+                .AddSingleton(ConfigProvider.GetConfig<IReceiptConfig>())
                 .AddModule(new DbModule());
 
             return builder;

src/Nethermind/Nethermind.Api/NethermindApi.cs

@@ -190,6 +190,7 @@ public ISealEngine SealEngine
         public ISyncModeSelector SyncModeSelector => ApiWithNetworkServiceContainer?.Resolve<ISyncModeSelector>()!;
 
         public ISyncProgressResolver? SyncProgressResolver => ApiWithNetworkServiceContainer?.Resolve<ISyncProgressResolver>();
+        public ISyncPointers? SyncPointers => ApiWithNetworkServiceContainer?.Resolve<ISyncPointers>();
         public IBetterPeerStrategy? BetterPeerStrategy { get; set; }
         public IPivot? Pivot { get; set; }
         public ISyncPeerPool? SyncPeerPool => ApiWithNetworkServiceContainer?.Resolve<ISyncPeerPool>();

src/Nethermind/Nethermind.Benchmark.Runner/Program.cs

@@ -26,6 +26,7 @@ public DashboardConfig(params Job[] jobs)
             AddColumnProvider(BenchmarkDotNet.Columns.DefaultColumnProviders.Descriptor);
             AddColumnProvider(BenchmarkDotNet.Columns.DefaultColumnProviders.Statistics);
             AddColumnProvider(BenchmarkDotNet.Columns.DefaultColumnProviders.Params);
+            AddColumnProvider(BenchmarkDotNet.Columns.DefaultColumnProviders.Metrics);
             AddLogger(BenchmarkDotNet.Loggers.ConsoleLogger.Default);
             AddExporter(BenchmarkDotNet.Exporters.Json.JsonExporter.FullCompressed);
             AddDiagnoser(BenchmarkDotNet.Diagnosers.MemoryDiagnoser.Default);
@@ -59,7 +60,7 @@ public static void Main(string[] args)
             {
                 foreach (Assembly assembly in additionalJobAssemblies)
                 {
-                    BenchmarkRunner.Run(assembly, new DashboardConfig(Job.MediumRun.WithRuntime(CoreRuntime.Core80)), args);
+                    BenchmarkRunner.Run(assembly, new DashboardConfig(Job.MediumRun.WithRuntime(CoreRuntime.Core90)), args);
                 }
 
                 foreach (Assembly assembly in simpleJobAssemblies)

src/Nethermind/Nethermind.Benchmark/Core/ParallelBenchmark.cs

@@ -0,0 +1,59 @@
+// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using BenchmarkDotNet.Attributes;
+using Nethermind.Core.Threading;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Nethermind.Benchmarks.Core;
+
+[HideColumns("Job", "RatioSD")]
+public class ParallelBenchmark
+{
+    private int[] _times;
+
+    [GlobalSetup]
+    public void Setup()
+    {
+        _times = new int[200];
+
+        for (int i = 0; i < _times.Length; i++)
+        {
+            _times[i] = i % 100;
+        }
+    }
+
+    [Benchmark(Baseline = true)]
+    public void ParallelFor()
+    {
+        Parallel.For(
+            0,
+            _times.Length,
+            (i) => Thread.Sleep(_times[i]));
+    }
+
+    [Benchmark]
+    public void ParallelForEach()
+    {
+        Parallel.ForEach(
+            _times,
+            (time) => Thread.Sleep(time));
+    }
+
+    [Benchmark]
+    public void UnbalancedParallel()
+    {
+        ParallelUnbalancedWork.For<int[]>(
+            0,
+            _times.Length,
+            ParallelUnbalancedWork.DefaultOptions,
+            _times,
+            (i, value) =>
+            {
+                Thread.Sleep(value[i]);
+                return value;
+            },
+            (value) => { });
+    }
+}