Optimize Tasks collections in .Net 9

Dockerfile

@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
 # SPDX-License-Identifier: LGPL-3.0-only
 
-FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0-noble AS build
+FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:9.0-noble AS build
 
 ARG BUILD_CONFIG=release
 ARG BUILD_TIMESTAMP
@@ -18,7 +18,7 @@ RUN arch=$([ "$TARGETARCH" = "amd64" ] && echo "x64" || echo "$TARGETARCH") && \
 # A temporary symlink to support the old executable name
 RUN ln -s -r /publish/nethermind /publish/Nethermind.Runner
 
-FROM --platform=$TARGETPLATFORM mcr.microsoft.com/dotnet/aspnet:8.0-noble
+FROM --platform=$TARGETPLATFORM mcr.microsoft.com/dotnet/aspnet:9.0-noble
 
 WORKDIR /nethermind
 

Dockerfile.chiseled

@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited
 # SPDX-License-Identifier: LGPL-3.0-only
 
-FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0-noble AS build
+FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:9.0-noble AS build
 
 ARG BUILD_CONFIG=release
 ARG BUILD_TIMESTAMP
@@ -21,7 +21,7 @@ RUN cd /publish && \
   mkdir logs && \
   mkdir nethermind_db
 
-FROM --platform=$TARGETPLATFORM mcr.microsoft.com/dotnet/aspnet:8.0-noble-chiseled
+FROM --platform=$TARGETPLATFORM mcr.microsoft.com/dotnet/aspnet:9.0-noble-chiseled
 
 WORKDIR /nethermind
 

Dockerfile.diag

@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
 # SPDX-License-Identifier: LGPL-3.0-only
 
-FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0-noble AS build
+FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:9.0-noble AS build
 
 ARG BUILD_CONFIG=release
 ARG BUILD_TIMESTAMP
@@ -22,7 +22,7 @@ RUN dotnet tool install -g dotnet-dump && \
     dotnet tool install -g dotnet-trace && \
     dotnet tool install -g JetBrains.dotTrace.GlobalTools
 
-FROM --platform=$TARGETPLATFORM mcr.microsoft.com/dotnet/aspnet:8.0-noble
+FROM --platform=$TARGETPLATFORM mcr.microsoft.com/dotnet/aspnet:9.0-noble
 
 WORKDIR /nethermind
 

global.json

@@ -1,7 +1,7 @@
 {
   "sdk": {
-    "version": "8.0.0",
-    "allowPrerelease": false,
+    "version": "9.0.0",
+    "allowPrerelease": true,
     "rollForward": "latestFeature"
   }
 }

src/Nethermind/Directory.Build.props

@@ -4,8 +4,8 @@
     <Configuration Condition="'$(Configuration)' == ''">Debug</Configuration>
     <ContinuousIntegrationBuild Condition="'$(CI)' == 'true'">true</ContinuousIntegrationBuild>
     <LangVersion>latest</LangVersion>
-    <TargetFramework>net8.0</TargetFramework>
-    <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
+    <TargetFramework>net9.0</TargetFramework>
+    <!--<TreatWarningsAsErrors>true</TreatWarningsAsErrors>-->
     <UseArtifactsOutput>true</UseArtifactsOutput>
   </PropertyGroup>
 

src/Nethermind/Directory.Packages.props

@@ -26,8 +26,8 @@
     <PackageVersion Include="HexMate" Version="0.0.3" />
     <PackageVersion Include="Jint" Version="2.11.58" />
     <PackageVersion Include="MathNet.Numerics.FSharp" Version="5.0.0" />
-    <PackageVersion Include="Microsoft.AspNetCore.DataProtection" Version="8.0.7" />
-    <PackageVersion Include="Microsoft.AspNetCore.DataProtection.Extensions" Version="8.0.7" />
+    <PackageVersion Include="Microsoft.AspNetCore.DataProtection" Version="9.0.0-rc.1.24452.1" />
+    <PackageVersion Include="Microsoft.AspNetCore.DataProtection.Extensions" Version="9.0.0-rc.1.24452.1" />
     <PackageVersion Include="Microsoft.Build.Tasks.Git" Version="8.0.0" />
     <PackageVersion Include="Microsoft.ClearScript.V8" Version="7.4.5" />
     <PackageVersion Include="Microsoft.ClearScript.V8.Native.linux-arm64" Version="7.4.5" />
@@ -37,9 +37,9 @@
     <PackageVersion Include="Microsoft.ClearScript.V8.Native.win-x64" Version="7.4.5" />
     <PackageVersion Include="McMaster.Extensions.CommandLineUtils" Version="4.1.1" />
     <PackageVersion Include="Microsoft.Extensions.Logging" Version="8.0.0" />
-    <PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.1" />
-    <PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="8.0.0" />
-    <PackageVersion Include="Microsoft.Extensions.ObjectPool" Version="8.0.7" />
+    <PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.0-rc.1.24431.7" />
+    <PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="9.0.0-rc.1.24431.7" />
+    <PackageVersion Include="Microsoft.Extensions.ObjectPool" Version="9.0.0-rc.1.24452.1" />
     <PackageVersion Include="Microsoft.IdentityModel.JsonWebTokens" Version="8.0.1" />
     <PackageVersion Include="Microsoft.IO.RecyclableMemoryStream" Version="3.0.1" />
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.10.0" />
@@ -72,10 +72,10 @@
     <PackageVersion Include="SCrypt" Version="2.0.0.2" />
     <PackageVersion Include="Shouldly" Version="4.2.1" />
     <PackageVersion Include="Snappier" Version="1.1.6" />
-    <PackageVersion Include="System.Configuration.ConfigurationManager" Version="8.0.0" />
-    <PackageVersion Include="System.IO.Pipelines" Version="8.0.0" />
+    <PackageVersion Include="System.Configuration.ConfigurationManager" Version="9.0.0-rc.1.24431.7" />
+    <PackageVersion Include="System.IO.Pipelines" Version="9.0.0-rc.1.24431.7" />
     <PackageVersion Include="System.Linq.Async" Version="6.0.1" />
-    <PackageVersion Include="System.Security.Cryptography.ProtectedData" Version="8.0.0" />
+    <PackageVersion Include="System.Security.Cryptography.ProtectedData" Version="9.0.0-rc.1.24431.7" />
     <PackageVersion Include="TestableIO.System.IO.Abstractions.TestingHelpers" Version="21.0.29" />
     <PackageVersion Include="TestableIO.System.IO.Abstractions.Wrappers" Version="21.0.29" />
     <PackageVersion Include="Websocket.Client" Version="5.1.2" />

src/Nethermind/Nethermind.Consensus/Processing/BlockchainProcessor.cs

@@ -167,7 +167,7 @@ public async Task StopAsync(bool processRemainingBlocks = false)
             _blockQueue.CompleteAdding();
         }
 
-        await Task.WhenAll((_recoveryTask ?? Task.CompletedTask), (_processorTask ?? Task.CompletedTask));
+        await Task.WhenAll(_recoveryTask ?? Task.CompletedTask, _processorTask ?? Task.CompletedTask);
         if (_logger.IsInfo) _logger.Info("Blockchain Processor shutdown complete.. please wait for all components to close");
     }
 

src/Nethermind/Nethermind.Consensus/Producers/MultipleBlockProducer.cs

@@ -7,6 +7,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 using Nethermind.Core;
+using Nethermind.Core.Collections;
 using Nethermind.Evm.Tracing;
 using Nethermind.Logging;
 
@@ -32,23 +33,20 @@ protected MultipleBlockProducer(
         public async Task<Block?> BuildBlock(BlockHeader? parentHeader, IBlockTracer? blockTracer = null,
             PayloadAttributes? payloadAttributes = null, CancellationToken? token = null)
         {
-            Task<Block?>[] produceTasks = new Task<Block?>[_blockProducers.Length];
+            using ArrayPoolList<Task<Block>> produceTasks = new(_blockProducers.Length);
             for (int i = 0; i < _blockProducers.Length; i++)
             {
                 T blockProducerInfo = _blockProducers[i];
-                if (!blockProducerInfo.Condition.CanProduce(parentHeader))
-                {
-                    produceTasks[i] = Task.FromResult<Block?>(null);
-                    continue;
-                }
-                produceTasks[i] = blockProducerInfo.BlockProducer.BuildBlock(parentHeader, blockProducerInfo.BlockTracer, cancellationToken: token);
+                produceTasks.Add(!blockProducerInfo.Condition.CanProduce(parentHeader!)
+                    ? Task.FromResult<Block?>(null)
+                    : blockProducerInfo.BlockProducer.BuildBlock(parentHeader, blockProducerInfo.BlockTracer, cancellationToken: token));
             }
 
             IEnumerable<(Block? Block, T BlockProducer)> blocksWithProducers;
 
             try
             {
-                Block?[] blocks = await Task.WhenAll(produceTasks);
+                Block?[] blocks = await Task.WhenAll<Block?>(produceTasks.AsSpan());
                 blocksWithProducers = blocks.Zip(_blockProducers);
             }
             catch (OperationCanceledException)

src/Nethermind/Nethermind.Core.Test/Encoding/TxDecoderTests.cs

@@ -7,6 +7,7 @@
 using System.Threading.Tasks;
 using FluentAssertions;
 using FluentAssertions.Numeric;
+using Nethermind.Core.Collections;
 using Nethermind.Core.Crypto;
 using Nethermind.Core.Eip2930;
 using Nethermind.Core.Extensions;
@@ -68,7 +69,7 @@ public class TxDecoderTests
 
         [TestCaseSource(nameof(TestCaseSource))]
         [Repeat(10)] // Might wanna increase this to double check when changing logic as on lower value, it does not reproduce.
-        public void CanCorrectlyCalculateTxHash_when_called_concurrently((Transaction Tx, string Description) testCase)
+        public async Task CanCorrectlyCalculateTxHash_when_called_concurrently((Transaction Tx, string Description) testCase)
         {
             Transaction tx = testCase.Tx;
 
@@ -80,14 +81,12 @@ public void CanCorrectlyCalculateTxHash_when_called_concurrently((Transaction Tx
 
             decodedTx.SetPreHash(rlp.Bytes);
 
-            IEnumerable<Task<AndConstraint<ComparableTypeAssertions<Hash256>>>> tasks = Enumerable
+            using ArrayPoolList<Task<AndConstraint<ComparableTypeAssertions<Hash256>>>> tasks = Enumerable
                 .Range(0, 32)
-                .Select((_) =>
-                    Task.Factory
-                        .StartNew(() => decodedTx.Hash.Should().Be(expectedHash),
-                            TaskCreationOptions.RunContinuationsAsynchronously));
+                .Select(_ => Task.Factory.StartNew(() => decodedTx.Hash.Should().Be(expectedHash), TaskCreationOptions.RunContinuationsAsynchronously))
+                .ToPooledList(32);
 
-            Task.WaitAll(tasks.ToArray());
+            await Task.WhenAll<AndConstraint<ComparableTypeAssertions<Hash256>>>(tasks.AsSpan());
         }
 
         [TestCaseSource(nameof(TestCaseSource))]

src/Nethermind/Nethermind.Core/PubSub/CompositePublisher.cs

@@ -1,34 +1,28 @@
 // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
 // SPDX-License-Identifier: LGPL-3.0-only
 
+using System;
 using System.Threading.Tasks;
+using Nethermind.Core.Collections;
 
 namespace Nethermind.Core.PubSub
 {
-    public class CompositePublisher : IPublisher
+    public class CompositePublisher(params IPublisher[] publishers) : IPublisher
     {
-        private readonly IPublisher[] _publishers;
-
-        public CompositePublisher(params IPublisher[] publishers)
-        {
-            _publishers = publishers;
-        }
-
         public async Task PublishAsync<T>(T data) where T : class
         {
-            // TODO: .Net 9 stackalloc
-            Task[] tasks = new Task[_publishers.Length];
-            for (int i = 0; i < _publishers.Length; i++)
+            using ArrayPoolList<Task> tasks = new(publishers.Length);
+            for (int i = 0; i < publishers.Length; i++)
             {
-                tasks[i] = _publishers[i].PublishAsync(data);
+                tasks.Add(publishers[i].PublishAsync(data));
             }
 
-            await Task.WhenAll(tasks);
+            await Task.WhenAll(tasks.AsSpan());
         }
 
         public void Dispose()
         {
-            foreach (IPublisher publisher in _publishers)
+            foreach (IPublisher publisher in publishers)
             {
                 publisher.Dispose();
             }

src/Nethermind/Nethermind.Db/RocksDbInitializer.cs

@@ -4,6 +4,7 @@
 using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;
+using Nethermind.Core.Collections;
 
 namespace Nethermind.Db
 {
@@ -68,13 +69,13 @@ protected void InitAll()
 
         protected async Task InitAllAsync()
         {
-            HashSet<Task> allInitializers = new();
-            foreach (var registration in _registrations)
+            using ArrayPoolList<Task> allInitializers = new(_registrations.Count);
+            foreach (Action registration in _registrations)
             {
                 allInitializers.Add(Task.Run(() => registration.Invoke()));
             }
 
-            await Task.WhenAll(allInitializers);
+            await Task.WhenAll(allInitializers.AsSpan());
         }
 
         protected static string GetTitleDbName(string dbName) => char.ToUpper(dbName[0]) + dbName[1..];

src/Nethermind/Nethermind.Evm.Test/Tracing/GethLikeCallTracerTests.cs

@@ -17,7 +17,7 @@ namespace Nethermind.Evm.Test.Tracing;
 [TestFixture]
 public class GethLikeCallTracerTests : VirtualMachineTestsBase
 {
-    private static readonly JsonSerializerOptions SerializerOptions = EthereumJsonSerializer.JsonOptionsIndented;
+    private static readonly JsonSerializerOptions SerializerOptions = new(EthereumJsonSerializer.JsonOptionsIndented) { NewLine = "\n" };
     private const string? WithLog = """{"withLog":true}""";
     private const string? OnlyTopCall = """{"onlyTopCall":true}""";
     private const string? WithLogAndOnlyTopCall = """{"withLog":true,"onlyTopCall":true}""";
@@ -27,14 +27,8 @@ private string ExecuteCallTrace(byte[] code, string? tracerConfig = null)
         (_, Transaction tx) = PrepareTx(MainnetSpecProvider.CancunActivation, 100000, code);
         NativeCallTracer tracer = new(tx, GetGethTraceOptions(tracerConfig));
 
-        GethLikeTxTrace callTrace = Execute(
-                tracer,
-                code,
-                MainnetSpecProvider.CancunActivation)
-            .BuildResult();
-        return JsonSerializer.Serialize(callTrace.CustomTracerResult?.Value, SerializerOptions)
-            // fix for windows, can be done better in .NET 9: https://github.com/dotnet/runtime/issues/84117
-            .ReplaceLineEndings("\n");
+        GethLikeTxTrace callTrace = Execute(tracer, code, MainnetSpecProvider.CancunActivation).BuildResult();
+        return JsonSerializer.Serialize(callTrace.CustomTracerResult?.Value, SerializerOptions);
     }
 
     private static GethTraceOptions GetGethTraceOptions(string? config) => GethTraceOptions.Default with

src/Nethermind/Nethermind.ExternalSigner.Plugin/Nethermind.ExternalSigner.Plugin.csproj

@@ -1,7 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/Nethermind/Nethermind.Network.Discovery/NodesLocator.cs

@@ -3,7 +3,9 @@
 
 using System.Text;
 using Nethermind.Core;
+using Nethermind.Core.Collections;
 using Nethermind.Core.Crypto;
+using Nethermind.Core.Extensions;
 using Nethermind.Logging;
 using Nethermind.Network.Discovery.Lifecycle;
 using Nethermind.Network.Discovery.Messages;
@@ -128,8 +130,8 @@ public async Task LocateNodesAsync(byte[]? searchedNodeId, CancellationToken can
                 int count = failRequestCount > 0 ? failRequestCount : _discoveryConfig.Concurrency;
                 IEnumerable<Node> nodesToSend = tryCandidates.Skip(nodesTriedCount).Take(count);
 
-                IEnumerable<Task<Result>> sendFindNodeTasks = SendFindNodes(searchedNodeId, nodesToSend, alreadyTriedNodes);
-                Result?[] results = await Task.WhenAll(sendFindNodeTasks);
+                using ArrayPoolList<Task<Result>> sendFindNodeTasks = SendFindNodes(searchedNodeId, nodesToSend, alreadyTriedNodes).ToPooledList(count);
+                Result[] results = await Task.WhenAll<Result>(sendFindNodeTasks.AsSpan());
 
                 if (results.Length == 0)
                 {

src/Nethermind/Nethermind.Network/CompositeNodeSource.cs

@@ -8,6 +8,8 @@
 using System.Threading;
 using System.Threading.Channels;
 using System.Threading.Tasks;
+using Nethermind.Core.Collections;
+using Nethermind.Core.Extensions;
 using Nethermind.Stats.Model;
 
 namespace Nethermind.Network;
@@ -20,14 +22,13 @@ public async IAsyncEnumerable<Node> DiscoverNodes([EnumeratorCancellation] Cance
     {
         Channel<Node> ch = Channel.CreateBounded<Node>(1);
 
-        // TODO: .Net 9 stackalloc
-        Task[] feedTasks = _nodeSources.Select(async innerSource =>
+        using ArrayPoolList<Task> feedTasks = _nodeSources.Select(async innerSource =>
         {
             await foreach (Node node in innerSource.DiscoverNodes(cancellationToken))
             {
                 await ch.Writer.WriteAsync(node, cancellationToken);
             }
-        }).ToArray();
+        }).ToPooledList(_nodeSources.Length * 16);
 
         try
         {
@@ -38,7 +39,7 @@ public async IAsyncEnumerable<Node> DiscoverNodes([EnumeratorCancellation] Cance
         }
         finally
         {
-            await Task.WhenAll(feedTasks);
+            await Task.WhenAll(feedTasks.AsSpan());
         }
     }
 

src/Nethermind/Nethermind.Network/PeerManager.cs

@@ -12,7 +12,9 @@
 using FastEnumUtility;
 using Nethermind.Core;
 using Nethermind.Core.Attributes;
+using Nethermind.Core.Collections;
 using Nethermind.Core.Crypto;
+using Nethermind.Core.Extensions;
 using Nethermind.Logging;
 using Nethermind.Network.Config;
 using Nethermind.Network.P2P;
@@ -206,7 +208,7 @@ private class CandidateSelection
         private async Task RunPeerUpdateLoop()
         {
             Channel<Peer> taskChannel = Channel.CreateBounded<Peer>(1);
-            List<Task>? tasks = Enumerable.Range(0, _outgoingConnectParallelism).Select(async (idx) =>
+            using ArrayPoolList<Task> tasks = Enumerable.Range(0, _outgoingConnectParallelism).Select(async idx =>
             {
                 await foreach (Peer peer in taskChannel.Reader.ReadAllAsync(_cancellationTokenSource.Token))
                 {
@@ -226,7 +228,7 @@ private async Task RunPeerUpdateLoop()
                     }
                 }
                 if (_logger.IsDebug) _logger.Debug($"Connect worker {idx} completed");
-            }).ToList();
+            }).ToPooledList(_outgoingConnectParallelism);
 
             int loopCount = 0;
             long previousActivePeersCount = 0;
@@ -359,7 +361,7 @@ private async Task RunPeerUpdateLoop()
             }
 
             taskChannel.Writer.Complete();
-            await Task.WhenAll(tasks);
+            await Task.WhenAll(tasks.AsSpan());
         }
 
         private bool EnsureAvailableActivePeerSlot()