From 584b3c2220b190971658e722153a80e303d56dca Mon Sep 17 00:00:00 2001 From: BrennanConroy Date: Mon, 5 Mar 2018 12:49:49 -0800 Subject: [PATCH] Add SignalR Client and scenario (#398) --- build/dependencies.props | 5 +- src/Benchmarks.ClientJob/Latency.cs | 10 +- src/Benchmarks.ClientJob/Worker.cs | 3 +- src/Benchmarks/Benchmarks.csproj | 2 + src/Benchmarks/Configuration/Scenarios.cs | 3 + .../Middleware/SignalRMiddleware.cs | 45 +++ src/Benchmarks/Startup.cs | 11 + src/Benchmarks/signalr.json | 13 + src/BenchmarksClient/Startup.cs | 11 +- src/BenchmarksDriver/Program.cs | 10 +- src/BenchmarksDriver/README.md | 5 + .../BenchmarksWorkers.csproj | 2 + src/BenchmarksWorkers/WorkerFactory.cs | 4 + .../Workers/SignalRSerializer.cs | 283 +++++++++++++++ .../Workers/SignalRWorker.cs | 321 ++++++++++++++++++ .../Workers/WrkSerializer.cs | 3 +- 16 files changed, 716 insertions(+), 15 deletions(-) create mode 100644 src/Benchmarks/Middleware/SignalRMiddleware.cs create mode 100644 src/Benchmarks/signalr.json create mode 100644 src/BenchmarksWorkers/Workers/SignalRSerializer.cs create mode 100644 src/BenchmarksWorkers/Workers/SignalRWorker.cs diff --git a/build/dependencies.props b/build/dependencies.props index 06ab27c33..a2c277d4a 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -1,14 +1,15 @@ - 2.1.0-* + 2.1.0-preview2-3* 4.4.0-* 2.1.1-* 10.0.1 2.15.4 + 1.0.0-* 2.0.0 2.0.0 2.0.0 - 2.1.0-preview1-26016-05 + 2.1.0-preview2-26225-03 2.0.0-* netcoreapp2.0 diff --git a/src/Benchmarks.ClientJob/Latency.cs b/src/Benchmarks.ClientJob/Latency.cs index 3cade2ba6..2baecc5db 100644 --- a/src/Benchmarks.ClientJob/Latency.cs +++ b/src/Benchmarks.ClientJob/Latency.cs @@ -5,10 +5,10 @@ namespace Benchmarks.ClientJob { public class Latency { - public double Average { get; set; } - public double Within50thPercentile { get; set; } - public double Within75thPercentile { get; set; } - public double Within90thPercentile { get; set; } - public double Within99thPercentile { get; set; } + public double Average { get; set; } = -1; + public double Within50thPercentile { get; set; } = -1; + public double Within75thPercentile { get; set; } = -1; + public double Within90thPercentile { get; set; } = -1; + public double Within99thPercentile { get; set; } = -1; } } diff --git a/src/Benchmarks.ClientJob/Worker.cs b/src/Benchmarks.ClientJob/Worker.cs index 4d9703406..bbbc39a1a 100644 --- a/src/Benchmarks.ClientJob/Worker.cs +++ b/src/Benchmarks.ClientJob/Worker.cs @@ -5,6 +5,7 @@ namespace Benchmarks.ClientJob { public enum Worker { - Wrk + Wrk, + SignalR, } } diff --git a/src/Benchmarks/Benchmarks.csproj b/src/Benchmarks/Benchmarks.csproj index 796b1186d..33f5b8aab 100644 --- a/src/Benchmarks/Benchmarks.csproj +++ b/src/Benchmarks/Benchmarks.csproj @@ -32,6 +32,8 @@ + + diff --git a/src/Benchmarks/Configuration/Scenarios.cs b/src/Benchmarks/Configuration/Scenarios.cs index d365430bf..9f5e39a76 100644 --- a/src/Benchmarks/Configuration/Scenarios.cs +++ b/src/Benchmarks/Configuration/Scenarios.cs @@ -151,6 +151,9 @@ public Scenarios(IScenariosConfiguration scenariosConfiguration) [ScenarioPath("/mvc/fortunes/dapper")] public bool MvcDbFortunesDapper { get; set; } + [ScenarioPath("/signalr/broadcast")] + public bool SignalRBroadcast { get; set; } + public bool Any(string partialName) => typeof(Scenarios).GetTypeInfo().DeclaredProperties .Where(p => p.Name.IndexOf(partialName, StringComparison.Ordinal) >= 0 && (bool)p.GetValue(this)) diff --git a/src/Benchmarks/Middleware/SignalRMiddleware.cs b/src/Benchmarks/Middleware/SignalRMiddleware.cs new file mode 100644 index 000000000..37480f1c0 --- /dev/null +++ b/src/Benchmarks/Middleware/SignalRMiddleware.cs @@ -0,0 +1,45 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Benchmarks.Configuration; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.SignalR; + +namespace Benchmarks.Middleware +{ + public static class SignalRMiddlewareExtensions + { + public static IApplicationBuilder UseSignalRMiddleware(this IApplicationBuilder builder) + { + return builder.UseSignalR(route => + { + route.MapHub(Scenarios.GetPath(s => s.SignalRBroadcast) + "/default"); + }); + } + } + + public class EchoHub : Hub + { + public async Task Echo(int duration) + { + try + { + var t = new CancellationTokenSource(); + t.CancelAfter(TimeSpan.FromSeconds(duration)); + while (!t.IsCancellationRequested && !Context.Connection.ConnectionAbortedToken.IsCancellationRequested) + { + await Clients.All.SendAsync("echo", DateTime.UtcNow); + } + } + catch (Exception e) + { + Console.WriteLine(e); + } + + Console.WriteLine("Echo exited"); + } + } +} \ No newline at end of file diff --git a/src/Benchmarks/Startup.cs b/src/Benchmarks/Startup.cs index 76d032f53..bc1a87002 100644 --- a/src/Benchmarks/Startup.cs +++ b/src/Benchmarks/Startup.cs @@ -154,6 +154,12 @@ public void ConfigureServices(IServiceCollection services) { services.AddResponseCaching(); } + + if (Scenarios.Any("SignalR")) + { + services.AddSignalR() + .AddMessagePackProtocol(); + } } public void Configure(IApplicationBuilder app, IHostingEnvironment env) @@ -307,6 +313,11 @@ public void Configure(IApplicationBuilder app, IHostingEnvironment env) app.UseResponseCachingPlaintextVaryByCached(); } + if (Scenarios.SignalRBroadcast) + { + app.UseSignalRMiddleware(); + } + app.RunDebugInfoPage(); } } diff --git a/src/Benchmarks/signalr.json b/src/Benchmarks/signalr.json new file mode 100644 index 000000000..6207c06af --- /dev/null +++ b/src/Benchmarks/signalr.json @@ -0,0 +1,13 @@ +{ + "Default": { + "ClientName": "signalr", + "Source": { + "Repository": "https://github.com/aspnet/benchmarks.git", + "BranchOrCommit": "brecon/signalrClientNew", + "Project": "src/Benchmarks/Benchmarks.csproj" + } + }, + "SignalRBroadcast": { + "Path": "/signalr/broadcast" + } +} \ No newline at end of file diff --git a/src/BenchmarksClient/Startup.cs b/src/BenchmarksClient/Startup.cs index c59ec7679..3cbf8cdc5 100644 --- a/src/BenchmarksClient/Startup.cs +++ b/src/BenchmarksClient/Startup.cs @@ -126,7 +126,7 @@ private static async Task ProcessJobs(CancellationToken cancellationToken) job.State = ClientState.Deleting; } else - { + { await worker.StartAsync(); } } @@ -151,15 +151,18 @@ private static async Task ProcessJobs(CancellationToken cancellationToken) } else if (job.State == ClientState.Deleting) { - Log($"Deleting job {worker.JobLogText}"); + Log($"Deleting job {worker?.JobLogText ?? "no worker found"}"); try { - await worker.StopAsync(); + if (worker != null) + { + await worker.StopAsync(); + } } finally { - worker.Dispose(); + worker?.Dispose(); worker = null; _jobs.Remove(job.Id); diff --git a/src/BenchmarksDriver/Program.cs b/src/BenchmarksDriver/Program.cs index 448f4a3a4..c22d64c45 100644 --- a/src/BenchmarksDriver/Program.cs +++ b/src/BenchmarksDriver/Program.cs @@ -477,11 +477,19 @@ public static int Main(string[] args) mergedClientJob.Merge(job); _clientJob = mergedClientJob.ToObject(); - if (clientNameOption.HasValue() && Enum.TryParse(clientNameOption.Value(), ignoreCase: true, result: out var worker)) + if (clientNameOption.HasValue()) { + if (!Enum.TryParse(clientNameOption.Value(), ignoreCase: true, result: out var worker)) + { + Log($"Could not find worker {clientNameOption.Value()}"); + return 9; + } + _clientJob.Client = worker; } + Log($"Using worker {_clientJob.Client}"); + // Override default ClientJob settings if options are set if (connectionsOption.HasValue()) { diff --git a/src/BenchmarksDriver/README.md b/src/BenchmarksDriver/README.md index 552fc5b52..3ff1bd9b3 100644 --- a/src/BenchmarksDriver/README.md +++ b/src/BenchmarksDriver/README.md @@ -56,6 +56,11 @@ Properties of the Wrk client ScriptName Name of the script used by wrk. PipelineDepth Depth of pipeline used by clients. + +Properties of the SignalR client + HubProtocol Name of the hub protocol to be used between client and server. + TransportType Name of the transport to communicate over. + LogLevel LogLevel name for SignalR connections to use. e.g. 'Trace' or 'Warning' ``` ### Examples diff --git a/src/BenchmarksWorkers/BenchmarksWorkers.csproj b/src/BenchmarksWorkers/BenchmarksWorkers.csproj index 8fc8d36f9..69836c92c 100644 --- a/src/BenchmarksWorkers/BenchmarksWorkers.csproj +++ b/src/BenchmarksWorkers/BenchmarksWorkers.csproj @@ -13,5 +13,7 @@ + + diff --git a/src/BenchmarksWorkers/WorkerFactory.cs b/src/BenchmarksWorkers/WorkerFactory.cs index 050716bab..e1c372075 100644 --- a/src/BenchmarksWorkers/WorkerFactory.cs +++ b/src/BenchmarksWorkers/WorkerFactory.cs @@ -19,6 +19,10 @@ static WorkerFactory() // Wrk Workers[Worker.Wrk] = clientJob => new WrkWorker(clientJob); ResultSerializers[Worker.Wrk] = () => new WrkSerializer(); + + // SignalR + Workers[Worker.SignalR] = clientJob => new SignalRWorker(clientJob); + ResultSerializers[Worker.SignalR] = () => new SignalRSerializer(); } static public IWorker CreateWorker(ClientJob clientJob) diff --git a/src/BenchmarksWorkers/Workers/SignalRSerializer.cs b/src/BenchmarksWorkers/Workers/SignalRSerializer.cs new file mode 100644 index 000000000..a1ec49412 --- /dev/null +++ b/src/BenchmarksWorkers/Workers/SignalRSerializer.cs @@ -0,0 +1,283 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Data.SqlClient; +using System.Linq; +using System.Threading.Tasks; +using Benchmarks.ClientJob; +using Benchmarks.ServerJob; +using Newtonsoft.Json; + +namespace BenchmarksWorkers.Workers +{ + class SignalRSerializer : IResultsSerializer + { + public async Task InitializeDatabaseAsync(string connectionString, string tableName) + { + string createCmd = + @" + IF OBJECT_ID(N'dbo." + tableName + @"', N'U') IS NULL + BEGIN + CREATE TABLE [dbo].[" + tableName + @"]( + [Id] [int] IDENTITY(1,1) NOT NULL PRIMARY KEY, + [Excluded] [bit] DEFAULT 0, + [DateTime] [datetimeoffset](7) NOT NULL, + [Session] [nvarchar](200) NOT NULL, + [Description] [nvarchar](200), + [AspNetCoreVersion] [nvarchar](50) NOT NULL, + [RuntimeVersion] [nvarchar](50) NOT NULL, + [Scenario] [nvarchar](50) NOT NULL, + [Hardware] [nvarchar](50) NOT NULL, + [HardwareVersion] [nvarchar](128) NOT NULL, + [OperatingSystem] [nvarchar](50) NOT NULL, + [Framework] [nvarchar](50) NOT NULL, + [RuntimeStore] [bit] NOT NULL, + [Scheme] [nvarchar](50) NOT NULL, + [Sources] [nvarchar](50) NULL, + [WebHost] [nvarchar](50) NOT NULL, + [Transport] [nvarchar](50) NOT NULL, + [HubProtocol] [nvarchar](50) NOT NULL, + [Connections] [int] NOT NULL, + [Duration] [int] NOT NULL, + [Path] [nvarchar](200) NULL, + [Headers] [nvarchar](max) NULL, + [Dimension] [nvarchar](50) NOT NULL, + [Value] [float] NOT NULL + ) + END + "; + + using (var connection = new SqlConnection(connectionString)) + { + await connection.OpenAsync(); + + using (var command = new SqlCommand(createCmd, connection)) + { + await command.ExecuteNonQueryAsync(); + } + } + } + + public async Task WriteJobResultsToSqlAsync(ServerJob serverJob, ClientJob clientJob, string connectionString, string tableName, string path, string session, string description, Statistics statistics, bool longRunning) + { + await RetryOnExceptionAsync(5, async () => + { + await WriteJobResultToSqlAsync(serverJob, clientJob, connectionString, tableName, path, session, description, statistics, longRunning, "RequestsPerSecond", statistics.RequestsPerSecond); + }); + + await RetryOnExceptionAsync(5, async () => + { + await WriteJobResultToSqlAsync(serverJob, clientJob, connectionString, tableName, path, session, description, statistics, longRunning, "CPU", statistics.Cpu); + }); + + await RetryOnExceptionAsync(5, async () => + { + await WriteJobResultToSqlAsync(serverJob, clientJob, connectionString, tableName, path, session, description, statistics, longRunning, "WorkingSet (MB)", statistics.WorkingSet); + }); + + if (statistics.LatencyAverage != -1) + { + await RetryOnExceptionAsync(5, async () => + { + await WriteJobResultToSqlAsync(serverJob, clientJob, connectionString, tableName, path, session, description, statistics, longRunning, "Latency Average (ms)", statistics.LatencyAverage); + }); + } + + if (statistics.Latency50Percentile != -1) + { + await RetryOnExceptionAsync(5, async () => + { + await WriteJobResultToSqlAsync(serverJob, clientJob, connectionString, tableName, path, session, description, statistics, longRunning, "Latency50Percentile (ms)", statistics.Latency50Percentile); + }); + } + + if (statistics.Latency75Percentile != -1) + { + await RetryOnExceptionAsync(5, async () => + { + await WriteJobResultToSqlAsync(serverJob, clientJob, connectionString, tableName, path, session, description, statistics, longRunning, "Latency75Percentile (ms)", statistics.Latency75Percentile); + }); + } + + if (statistics.Latency90Percentile != -1) + { + await RetryOnExceptionAsync(5, async () => + { + await WriteJobResultToSqlAsync(serverJob, clientJob, connectionString, tableName, path, session, description, statistics, longRunning, "Latency90Percentile (ms)", statistics.Latency90Percentile); + }); + } + + if (statistics.Latency99Percentile != -1) + { + await RetryOnExceptionAsync(5, async () => + { + await WriteJobResultToSqlAsync(serverJob, clientJob, connectionString, tableName, path, session, description, statistics, longRunning, "Latency99Percentile (ms)", statistics.Latency99Percentile); + }); + } + } + + private async Task WriteJobResultToSqlAsync(ServerJob serverJob, ClientJob clientJob, string connectionString, string tableName, string path, string session, string description, Statistics statistics, bool longRunning, string dimension, double value) + { + string insertCmd = + @" + INSERT INTO [dbo].[" + tableName + @"] + ([DateTime] + ,[Session] + ,[Description] + ,[AspNetCoreVersion] + ,[RuntimeVersion] + ,[Scenario] + ,[Hardware] + ,[HardwareVersion] + ,[OperatingSystem] + ,[Framework] + ,[RuntimeStore] + ,[Scheme] + ,[Sources] + ,[WebHost] + ,[Transport] + ,[HubProtocol] + ,[Connections] + ,[Duration] + ,[Path] + ,[Headers] + ,[Dimension] + ,[Value]) + VALUES + (@DateTime + ,@Session + ,@Description + ,@AspNetCoreVersion + ,@RuntimeVersion + ,@Scenario + ,@Hardware + ,@HardwareVersion + ,@OperatingSystem + ,@Framework + ,@RuntimeStore + ,@Scheme + ,@Sources + ,@WebHost + ,@Transport + ,@HubProtocol + ,@Connections + ,@Duration + ,@Path + ,@Headers + ,@Dimension + ,@Value) + "; + + using (var connection = new SqlConnection(connectionString)) + { + await connection.OpenAsync(); + + using (var command = new SqlCommand(insertCmd, connection)) + { + var p = command.Parameters; + p.AddWithValue("@DateTime", DateTimeOffset.UtcNow); + p.AddWithValue("@Session", session); + p.AddWithValue("@Description", description); + p.AddWithValue("@AspNetCoreVersion", serverJob.AspNetCoreVersion); + p.AddWithValue("@RuntimeVersion", serverJob.RuntimeVersion); + p.AddWithValue("@Scenario", serverJob.Scenario.ToString()); + p.AddWithValue("@Hardware", serverJob.Hardware.ToString()); + p.AddWithValue("@HardwareVersion", serverJob.HardwareVersion); + p.AddWithValue("@OperatingSystem", serverJob.OperatingSystem.ToString()); + p.AddWithValue("@Framework", "Core"); + p.AddWithValue("@RuntimeStore", serverJob.UseRuntimeStore); + p.AddWithValue("@Scheme", serverJob.Scheme.ToString().ToLowerInvariant()); + p.AddWithValue("@Sources", serverJob.ReferenceSources.Any() ? (object)ConvertToSqlString(serverJob.ReferenceSources) : DBNull.Value); + p.AddWithValue("@WebHost", serverJob.WebHost.ToString()); + p.AddWithValue("@Transport", clientJob.ClientProperties["TransportType"]); + p.AddWithValue("@HubProtocol", clientJob.ClientProperties["HubProtocol"]); + p.AddWithValue("@Connections", clientJob.Connections); + p.AddWithValue("@Duration", clientJob.Duration); + p.AddWithValue("@Path", string.IsNullOrEmpty(path) ? (object)DBNull.Value : path); + p.AddWithValue("@Headers", clientJob.Headers.Any() ? JsonConvert.SerializeObject(clientJob.Headers) : (object)DBNull.Value); + p.AddWithValue("@Dimension", dimension); + p.AddWithValue("@Value", value); + + await command.ExecuteNonQueryAsync(); + } + } + } + + private static string ConvertToSqlString(IEnumerable sources) + { + return string.Join(",", sources.Select(s => ConvertToSqlString(s))); + } + + private static string ConvertToSqlString(Source source) + { + const string aspnetPrefix = "https://github.com/aspnet/"; + const string gitSuffix = ".git"; + + var shortRepository = source.Repository; + + if (shortRepository.StartsWith(aspnetPrefix)) + { + shortRepository = shortRepository.Substring(aspnetPrefix.Length); + } + + if (shortRepository.EndsWith(gitSuffix)) + { + shortRepository = shortRepository.Substring(0, shortRepository.Length - gitSuffix.Length); + } + + if (string.IsNullOrEmpty(source.BranchOrCommit)) + { + return shortRepository; + } + else + { + return shortRepository + "@" + source.BranchOrCommit; + } + } + + private async static Task RetryOnExceptionAsync(int retries, Func operation, int milliSecondsDelay = 0) + { + var attempts = 0; + do + { + try + { + attempts++; + await operation(); + return; + } + catch (Exception e) + { + if (attempts == retries + 1) + { + throw; + } + + Log($"Attempt {attempts} failed: {e.Message}"); + + if (milliSecondsDelay > 0) + { + await Task.Delay(milliSecondsDelay); + } + } + } while (true); + } + + private static void Log(string message) + { + var time = DateTime.Now.ToString("hh:mm:ss.fff"); + Console.WriteLine($"[{time}] {message}"); + } + + public void ComputeAverages(Statistics average, IEnumerable samples) + { + // TODO: Do we want to do anything custom here + } + + public void Dispose() + { + } + } +} diff --git a/src/BenchmarksWorkers/Workers/SignalRWorker.cs b/src/BenchmarksWorkers/Workers/SignalRWorker.cs new file mode 100644 index 000000000..66dcf89ba --- /dev/null +++ b/src/BenchmarksWorkers/Workers/SignalRWorker.cs @@ -0,0 +1,321 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Net.Http; +using System.Net.WebSockets; +using System.Threading; +using System.Threading.Tasks; +using Benchmarks.ClientJob; +using Microsoft.AspNetCore.SignalR.Client; +using Microsoft.AspNetCore.Sockets; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; + +namespace BenchmarksWorkers.Workers +{ + public class SignalRWorker : IWorker + { + public string JobLogText { get; set; } + + private ClientJob _job; + private HttpClientHandler _httpClientHandler; + private List _connections; + private List _recvCallbacks; + private Timer _timer; + private List _requestsPerConnection; + private List> _latencyPerConnection; + private Stopwatch _workTimer = new Stopwatch(); + private bool _stopped; + private SemaphoreSlim _lock = new SemaphoreSlim(1); + + public SignalRWorker(ClientJob job) + { + _job = job; + + Debug.Assert(_job.Connections > 0, "There must be more than 0 connections"); + + // Configuring the http client to trust the self-signed certificate + _httpClientHandler = new HttpClientHandler + { + ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator + }; + + var jobLogText = + $"[ID:{_job.Id} Connections:{_job.Connections} Duration:{_job.Duration} Method:{_job.Method} ServerUrl:{_job.ServerBenchmarkUri}"; + + if (_job.Headers != null) + { + jobLogText += $" Headers:{JsonConvert.SerializeObject(_job.Headers)}"; + } + + TransportType transportType = default; + if (_job.ClientProperties.TryGetValue("TransportType", out var transport)) + { + transportType = Enum.Parse(transport); + jobLogText += $" TransportType:{transportType}"; + } + + jobLogText += "]"; + JobLogText = jobLogText; + + CreateConnections(transportType); + } + + public async Task StartAsync() + { + // start connections + var tasks = new List(_connections.Count); + foreach (var connection in _connections) + { + tasks.Add(connection.StartAsync()); + } + + await Task.WhenAll(tasks); + + _job.State = ClientState.Running; + _job.LastDriverCommunicationUtc = DateTime.UtcNow; + + // SendAsync will return as soon as the request has been sent (non-blocking) + await _connections[0].SendAsync("Echo", _job.Duration + 1); + _workTimer.Start(); + _timer = new Timer(StopClients, null, TimeSpan.FromSeconds(_job.Duration), Timeout.InfiniteTimeSpan); + } + + private async void StopClients(object t) + { + try + { + _timer.Change(Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); + await StopAsync(); + } + finally + { + _job.State = ClientState.Completed; + } + } + + public async Task StopAsync() + { + if (!await _lock.WaitAsync(0)) + { + // someone else is stopping, we only need to do it once + return; + } + try + { + if (_timer != null) + { + _timer?.Dispose(); + _timer = null; + + foreach (var callback in _recvCallbacks) + { + // stops stat collection from happening quicker than StopAsync + // and we can do all the calculations while close is occurring + callback.Dispose(); + } + + _workTimer.Stop(); + + _stopped = true; + + // stop connections + Log("Stopping connections"); + var tasks = new List(_connections.Count); + foreach (var connection in _connections) + { + tasks.Add(connection.StopAsync()); + } + + CalculateStatistics(); + + await Task.WhenAll(tasks); + + // TODO: Remove when clients no longer take a long time to "cool down" + await Task.Delay(5000); + + Log("Stopped worker"); + } + } + finally + { + _lock.Release(); + } + } + + public void Dispose() + { + var tasks = new List(_connections.Count); + foreach (var connection in _connections) + { + tasks.Add(connection.DisposeAsync()); + } + + Task.WhenAll(tasks).GetAwaiter().GetResult(); + + _httpClientHandler.Dispose(); + } + + private void CreateConnections(TransportType transportType = TransportType.WebSockets) + { + _connections = new List(_job.Connections); + _requestsPerConnection = new List(_job.Connections); + _latencyPerConnection = new List>(_job.Connections); + + var hubConnectionBuilder = new HubConnectionBuilder() + .WithUrl(_job.ServerBenchmarkUri) + .WithMessageHandler(_httpClientHandler) + .WithTransport(transportType); + + if (_job.ClientProperties.TryGetValue("LogLevel", out var logLevel)) + { + if (Enum.TryParse(logLevel, ignoreCase: true, result: out var level)) + { + hubConnectionBuilder.WithConsoleLogger(level); + } + } + + if (_job.ClientProperties.TryGetValue("HubProtocol", out var protocolName)) + { + switch (protocolName) + { + case "messagepack": + hubConnectionBuilder.WithMessagePackProtocol(); + break; + case "json": + hubConnectionBuilder.WithJsonProtocol(); + break; + default: + throw new Exception($"{protocolName} is an invalid hub protocol name."); + } + } + else + { + hubConnectionBuilder.WithJsonProtocol(); + } + + foreach (var header in _job.Headers) + { + hubConnectionBuilder.WithHeader(header.Key, header.Value); + } + + _recvCallbacks = new List(_job.Connections); + for (var i = 0; i < _job.Connections; i++) + { + _requestsPerConnection.Add(0); + _latencyPerConnection.Add(new List()); + + var connection = hubConnectionBuilder.Build(); + _connections.Add(connection); + + // Capture the connection ID + var id = i; + // setup event handlers + _recvCallbacks.Add(connection.On("echo", utcNow => + { + // TODO: Collect all the things + _requestsPerConnection[id] += 1; + + var latency = DateTime.UtcNow - utcNow; + _latencyPerConnection[id].Add(latency.TotalMilliseconds); + })); + + connection.Closed += e => + { + if (!_stopped) + { + var error = $"Connection closed early: {e}"; + _job.Error += error; + Log(error); + } + }; + } + } + + private void CalculateStatistics() + { + // RPS + var totalRequests = 0; + var min = int.MaxValue; + var max = 0; + for (var i = 0; i < _requestsPerConnection.Count; i++) + { + totalRequests += _requestsPerConnection[i]; + + if (_requestsPerConnection[i] > max) + { + max = _requestsPerConnection[i]; + } + if (_requestsPerConnection[i] < min) + { + min = _requestsPerConnection[i]; + } + } + // Review: This could be interesting information, see the gap between most active and least active connection + // Ideally they should be within a couple percent of each other, but if they aren't something could be wrong + Log($"Least Requests per Connection: {min}"); + Log($"Most Requests per Connection: {max}"); + + var rps = (double)totalRequests / _workTimer.ElapsedMilliseconds * 1000; + Log($"Total RPS: {rps}"); + _job.RequestsPerSecond = rps; + + // Latency + CalculateLatency(); + } + + private void CalculateLatency() + { + var avg = new List(_latencyPerConnection.Count); + var totalAvg = 0.0; + for (var i = 0; i < _latencyPerConnection.Count; i++) + { + avg.Add(0.0); + for (var j = 0; j < _latencyPerConnection[i].Count; j++) + { + avg[i] += _latencyPerConnection[i][j]; + } + avg[i] /= _latencyPerConnection[i].Count; + Log($"Average latency for connection #{i}: {avg[i]}"); + + _latencyPerConnection[i].Sort(); + totalAvg += avg[i]; + } + + totalAvg /= avg.Count; + _job.Latency.Average = totalAvg; + + var allConnections = new List(); + foreach (var connectionLatency in _latencyPerConnection) + { + allConnections.AddRange(connectionLatency); + } + + // Review: Each connection can have different latencies, how do we want to deal with that? + // We could just combine them all and ignore the fact that they are different connections + // Or we could preserve the results for each one and record them separately + allConnections.Sort(); + _job.Latency.Within50thPercentile = GetPercentile(50, allConnections); + _job.Latency.Within75thPercentile = GetPercentile(75, allConnections); + _job.Latency.Within90thPercentile = GetPercentile(90, allConnections); + _job.Latency.Within99thPercentile = GetPercentile(99, allConnections); + } + + private double GetPercentile(int percent, List sortedData) + { + var i = (percent * sortedData.Count) / 100.0 + 0.5; + var fractionPart = i - Math.Truncate(i); + + return (1.0 - fractionPart) * sortedData[(int)Math.Truncate(i)] + fractionPart * sortedData[(int)Math.Ceiling(i)]; + } + + private static void Log(string message) + { + var time = DateTime.Now.ToString("hh:mm:ss.fff"); + Console.WriteLine($"[{time}] {message}"); + } + } +} diff --git a/src/BenchmarksWorkers/Workers/WrkSerializer.cs b/src/BenchmarksWorkers/Workers/WrkSerializer.cs index deed3d253..002c63fad 100644 --- a/src/BenchmarksWorkers/Workers/WrkSerializer.cs +++ b/src/BenchmarksWorkers/Workers/WrkSerializer.cs @@ -411,7 +411,7 @@ INSERT INTO [dbo].[" + tableName + @"] p.AddWithValue("@Session", session); p.AddWithValue("@Description", description); p.AddWithValue("@AspNetCoreVersion", aspnetCoreVersion); - p.AddWithValue("@RuntimeVersion", aspnetCoreVersion); + p.AddWithValue("@RuntimeVersion", runtimeVersion); p.AddWithValue("@Scenario", scenario.ToString()); p.AddWithValue("@Hardware", hardware.ToString()); p.AddWithValue("@HardwareVersion", hardwareVersion); @@ -499,7 +499,6 @@ private async static Task RetryOnExceptionAsync(int retries, Func operatio } while (true); } - public void Dispose() { }