From ddbc51f02e1076b639796ad0d813c34f61881994 Mon Sep 17 00:00:00 2001 From: Nathan Willoughby Date: Tue, 5 Dec 2023 19:56:17 +1000 Subject: [PATCH] . --- .../ClientScriptExecutionRetriesTimeout.cs | 50 +++++--- ...NonV1IsNotRetriedWhenRetriesAreDisabled.cs | 38 +++++-- ...iptExecutionScriptServiceV1IsNotRetried.cs | 39 +++++-- ...ontractAssertionBuilderExtensionMethods.cs | 107 +++++++----------- 4 files changed, 135 insertions(+), 99 deletions(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionRetriesTimeout.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionRetriesTimeout.cs index 3ee951e2b..7daa85097 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionRetriesTimeout.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionRetriesTimeout.cs @@ -119,7 +119,7 @@ public async Task WhenGetCapabilitiesFails_AndTakesLongerThanTheRetryDuration_Th await tcpConnectionUtilities.RestartTcpConnection(); // Sleep to make the initial RPC call take longer than the allowed retry duration - await Task.Delay(retryDuration + TimeSpan.FromSeconds(1)); + await Task.Delay(retryDuration + TimeSpan.FromSeconds(5)); // Kill the first GetCapabilities call to force the rpc call into retries responseMessageTcpKiller.KillConnectionOnNextResponse(); @@ -133,8 +133,11 @@ public async Task WhenGetCapabilitiesFails_AndTakesLongerThanTheRetryDuration_Th .Build(); var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); - Func action = async () => await executeScriptTask; - await action.Should().ThrowAsync(); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase, clientAndTentacle) + .ForScriptService(ScriptServiceOperation.GetCapabilities).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); capabilitiesMethodUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)).Started.Should().Be(1); scriptMethodUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(0, "Test should not have not proceeded past GetCapabilities"); @@ -195,8 +198,12 @@ public async Task WhenRpcRetriesTimeOut_DuringStartScript_TheRpcCallIsCancelled( var duration = Stopwatch.StartNew(); var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); - Func action = async () => await executeScriptTask; - await action.Should().ThrowAsync(); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase, clientAndTentacle) + .ForScriptService(ScriptServiceOperation.StartScript).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); + duration.Stop(); recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().BeGreaterOrEqualTo(2); @@ -244,8 +251,11 @@ public async Task WhenStartScriptFails_AndTakesLongerThanTheRetryDuration_TheCal .Build(); var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); - Func action = async () => await executeScriptTask; - await action.Should().ThrowAsync(); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase, clientAndTentacle) + .ForScriptService(ScriptServiceOperation.StartScript).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(0); @@ -310,8 +320,12 @@ public async Task WhenRpcRetriesTimeOut_DuringGetStatus_TheRpcCallIsCancelled(Te var duration = Stopwatch.StartNew(); var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); - Func action = async () => await executeScriptTask; - await action.Should().ThrowAsync(); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase, clientAndTentacle) + .ForScriptService(ScriptServiceOperation.GetStatus).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); + duration.Stop(); @@ -362,8 +376,11 @@ public async Task WhenGetStatusFails_AndTakesLongerThanTheRetryDuration_TheCallI .Build(); var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); - Func action = async () => await executeScriptTask; - await action.Should().ThrowAsync(); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase, clientAndTentacle) + .ForScriptService(ScriptServiceOperation.GetStatus).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(1); @@ -443,7 +460,11 @@ public async Task WhenRpcRetriesTimeOut_DuringCancelScript_TheRpcCallIsCancelled // We cancel script execution via the cancellation token. This should trigger the CancelScript RPC call to be made testCancellationTokenSource.Cancel(); - await action.Should().ThrowAsync(); + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase, clientAndTentacle) + .ForScriptService(ScriptServiceOperation.CancelScript).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); + duration.Stop(); recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); @@ -507,7 +528,10 @@ public async Task WhenCancelScriptFails_AndTakesLongerThanTheRetryDuration_TheCa // We cancel script execution via the cancellation token. This should trigger the CancelScript RPC call to be made testCancellationTokenSource.Cancel(); - await action.Should().ThrowAsync(); + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase, clientAndTentacle) + .ForScriptService(ScriptServiceOperation.CancelScript).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().BeGreaterOrEqualTo(1); diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptServiceNonV1IsNotRetriedWhenRetriesAreDisabled.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptServiceNonV1IsNotRetriedWhenRetriesAreDisabled.cs index 92b10cf89..5585bd471 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptServiceNonV1IsNotRetriedWhenRetriesAreDisabled.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptServiceNonV1IsNotRetriedWhenRetriesAreDisabled.cs @@ -4,7 +4,6 @@ using System.Threading; using System.Threading.Tasks; using FluentAssertions; -using Halibut; using NUnit.Framework; using Octopus.Tentacle.CommonTestUtils.Builders; using Octopus.Tentacle.Contracts; @@ -54,7 +53,12 @@ public async Task WhenNetworkFailureOccurs_DuringGetCapabilities_TheCallIsNotRet .WithScriptBody(new ScriptBuilder().Print("hello")).Build(); var logs = new List(); - Assert.ThrowsAsync(async () => await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken)); + var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase, clientTentacle) + .ForScriptService(ScriptServiceOperation.GetCapabilities).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); var allLogs = logs.JoinLogs(); @@ -104,7 +108,12 @@ public async Task WhenANetworkFailureOccurs_DuringStartScript_TheCallIsNotRetrie .Build(); var logs = new List(); - Assert.ThrowsAsync(async () => await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken)); + var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase, clientTentacle) + .ForScriptService(ScriptServiceOperation.StartScript).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); var allLogs = logs.JoinLogs(); @@ -155,10 +164,14 @@ public async Task WhenANetworkFailureOccurs_DuringGetStatus_TheCallIsNotRetried( .Print("AllDone")) .Build(); - List logs = new List(); - Logger.Information("Starting and waiting for script exec"); - Assert.ThrowsAsync(async () => await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken)); - Logger.Information("Exception thrown."); + var logs = new List(); + + var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase, clientTentacle) + .ForScriptService(ScriptServiceOperation.GetStatus).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); var allLogs = logs.JoinLogs(); @@ -219,8 +232,13 @@ public async Task WhenANetworkFailureOccurs_DuringCancelScript_TheCallIsNotRetri .Print("AllDone")) .Build(); - List logs = new List(); - Assert.ThrowsAsync(async () => await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, cts.Token)); + var logs = new List(); + var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, cts.Token); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase, clientTentacle) + .ForScriptService(ScriptServiceOperation.CancelScript).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); var allLogs = logs.JoinLogs(); @@ -268,7 +286,7 @@ public async Task WhenANetworkFailureOccurs_DuringCompleteScript_TheCallIsNotRet .Print("AllDone")) .Build(); - List logs = new List(); + var logs = new List(); await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); var allLogs = logs.JoinLogs(); diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptServiceV1IsNotRetried.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptServiceV1IsNotRetried.cs index e0ca00975..d38b1ba04 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptServiceV1IsNotRetried.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptServiceV1IsNotRetried.cs @@ -4,7 +4,6 @@ using System.Threading; using System.Threading.Tasks; using FluentAssertions; -using Halibut; using NUnit.Framework; using Octopus.Tentacle.CommonTestUtils.Builders; using Octopus.Tentacle.Contracts; @@ -55,8 +54,13 @@ public async Task WhenANetworkFailureOccurs_DuringStartScript_WithATentacleThatO .Print("AllDone")) .Build(); - List logs = new List(); - Assert.ThrowsAsync(async () => await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken)); + var logs = new List(); + var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase, clientTentacle) + .ForScriptService(ScriptServiceOperation.StartScript).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); // Let the script finish. File.WriteAllText(waitForFile, ""); @@ -109,10 +113,13 @@ public async Task WhenANetworkFailureOccurs_DuringGetStatus_WithATentacleThatOnl .Print("AllDone")) .Build(); - List logs = new List(); - Logger.Information("Starting and waiting for script exec"); - Assert.ThrowsAsync(async () => await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken)); - Logger.Information("Exception thrown."); + var logs = new List(); + var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase, clientTentacle) + .ForScriptService(ScriptServiceOperation.GetStatus).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); // Let the script finish. File.WriteAllText(waitForFile, ""); @@ -176,7 +183,12 @@ public async Task WhenANetworkFailureOccurs_DuringCancelScript_WithATentacleThat var logs = new List(); - Assert.ThrowsAsync(async () => await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, cts.Token)); + var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, cts.Token); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase, clientTentacle) + .ForScriptService(ScriptServiceOperation.CancelScript).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); var allLogs = logs.JoinLogs(); allLogs.Should().NotContain("AllDone"); @@ -219,13 +231,16 @@ public async Task WhenANetworkFailureOccurs_DuringCompleteScript_WithATentacleTh var startScriptCommand = new LatestStartScriptCommandBuilder().WithScriptBody(new ScriptBuilder().Print("hello")).Build(); - List logs = new List(); - Assert.ThrowsAsync(async () => await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken)); + var logs = new List(); + var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); - // We Can not verify what will be in the logs because of race conditions in tentacle. - // The last complete script which we fail might come back with the logs. + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase, clientTentacle) + .ForScriptService(ScriptServiceOperation.CompleteScript).Build(); + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); + // We Can not verify what will be in the logs because of race conditions in tentacle. + // The last complete script which we fail might come back with the logs. recordedUsages.For(nameof(IAsyncClientScriptService.CompleteScriptAsync)).LastException.Should().NotBeNull(); recordedUsages.For(nameof(IAsyncClientScriptService.StartScriptAsync)).Started.Should().Be(1); recordedUsages.For(nameof(IAsyncClientScriptService.CompleteScriptAsync)).Started.Should().Be(1); diff --git a/source/Octopus.Tentacle.Tests.Integration/Support/ExceptionContractAssertionBuilderExtensionMethods.cs b/source/Octopus.Tentacle.Tests.Integration/Support/ExceptionContractAssertionBuilderExtensionMethods.cs index 3656bb4df..63b03a498 100644 --- a/source/Octopus.Tentacle.Tests.Integration/Support/ExceptionContractAssertionBuilderExtensionMethods.cs +++ b/source/Octopus.Tentacle.Tests.Integration/Support/ExceptionContractAssertionBuilderExtensionMethods.cs @@ -3,7 +3,6 @@ using System.Threading.Tasks; using FluentAssertions; using FluentAssertions.Execution; -using FluentAssertions.Primitives; using FluentAssertions.Specialized; using Halibut; @@ -45,6 +44,8 @@ public static void ShouldMatchExceptionContract( string because = "", params object[] becauseArgs) { + using var scope = new AssertionScope(); + exception.Should().NotBeNull(because, becauseArgs); exception.ShouldBeOfType(expected.ExceptionTypes, because, becauseArgs); exception.Message.Should().ContainAny(expected.ExceptionMessageShouldContainAny, because, becauseArgs); @@ -63,41 +64,6 @@ public static void ShouldBeOfType(this Exception subject, Type[] values, string public class ExceptionContractAssertionBuilder { - // - RPC Retries not supported - // - RPC Retries Disabled - // - RPC Retries - First Try - // - RPC Retries - Retrying - - // Connecting - // Transferring - - // Connecting / Transferring Error e.g. connection timeout or transferring error - // - Get Capabilities - // - Start Script - // - Get Status - // - Cancel Script - // - Complete Script - // - Upload File - // - Download File - - // Cancelled - // - Get Capabilities - // - Start Script - // - Get Status - // - Cancel Script - // - Complete Script - // - Upload File - // - Download File - - // RPC Retries Timeout - // - Get Capabilities - // - Start Script - // - Get Status - // - Cancel Script - // - Complete Script - // - Upload File - // - Download File - readonly FailureScenario failureScenario; readonly TentacleConfigurationTestCase tentacleConfigurationTestCase; readonly ClientAndTentacle clientAndTentacle; @@ -142,37 +108,37 @@ public ExceptionContract Build() throw new InvalidOperationException("Script Service Version not specified in the TentacleConfigurationTestCase"); } - if (fileTransferServiceOperation != null) + //if (fileTransferServiceOperation != null) + //{ + if (failureScenario == FailureScenario.ConnectionFaulted) { - if (failureScenario == FailureScenario.ConnectionFaulted) + switch (tentacleConfigurationTestCase.TentacleType) { - switch (tentacleConfigurationTestCase.TentacleType) - { - case TentacleType.Listening: - return new ExceptionContract(typeof(HalibutClientException), new[] - { - $"An error occurred when sending a request to '{clientAndTentacle.ServiceEndPoint}/', after the request began: Attempted to read past the end of the stream.", - $"An error occurred when sending a request to '{clientAndTentacle.ServiceEndPoint}/', after the request began: Unable to write data to the transport connection: An established connection was aborted by the software in your host machine", - $"An error occurred when sending a request to '{clientAndTentacle.ServiceEndPoint}/', after the request began: Unable to read data from the transport connection: An established connection was aborted by the software in your host machine", - $"An error occurred when sending a request to '{clientAndTentacle.ServiceEndPoint}/', after the request began: Unable to write data to the transport connection: An existing connection was forcibly closed by the remote host", - $"An error occurred when sending a request to '{clientAndTentacle.ServiceEndPoint}/', before the request could begin: Connection refused" - }); - case TentacleType.Polling: - return new ExceptionContract(typeof(HalibutClientException), new[] - { - "Attempted to read past the end of the stream.", - "Unable to write data to the transport connection: An established connection was aborted by the software in your host machine", - "Unable to read data from the transport connection: An established connection was aborted by the software in your host machine", - "Connection refused", - "Unable to write data to the transport connection: Broken pipe", - "Unable to write data to the transport connection: An existing connection was forcibly closed by the remote host" - }); - default: - throw new ArgumentOutOfRangeException(); - } + case TentacleType.Listening: + return new ExceptionContract(typeof(HalibutClientException), new[] + { + $"An error occurred when sending a request to '{clientAndTentacle.ServiceEndPoint}/', after the request began: Attempted to read past the end of the stream.", + $"An error occurred when sending a request to '{clientAndTentacle.ServiceEndPoint}/', after the request began: Unable to write data to the transport connection: An established connection was aborted by the software in your host machine", + $"An error occurred when sending a request to '{clientAndTentacle.ServiceEndPoint}/', after the request began: Unable to read data from the transport connection: An established connection was aborted by the software in your host machine", + $"An error occurred when sending a request to '{clientAndTentacle.ServiceEndPoint}/', after the request began: Unable to write data to the transport connection: An existing connection was forcibly closed by the remote host", + $"An error occurred when sending a request to '{clientAndTentacle.ServiceEndPoint}/', before the request could begin: Connection refused" + }); + case TentacleType.Polling: + return new ExceptionContract(typeof(HalibutClientException), new[] + { + "Attempted to read past the end of the stream.", + "Unable to write data to the transport connection: An established connection was aborted by the software in your host machine", + "Unable to read data from the transport connection: An established connection was aborted by the software in your host machine", + "Connection refused", + "Unable to write data to the transport connection: Broken pipe", + "Unable to write data to the transport connection: An existing connection was forcibly closed by the remote host" + }); + default: + throw new ArgumentOutOfRangeException(); } } - else if (scriptServiceOperation != null) + //} + if (scriptServiceOperation != null) { if (failureScenario == FailureScenario.ScriptExecutionCancelled) { @@ -187,6 +153,18 @@ public ExceptionContract Build() "A task was canceled." // Cancellation during StartScript while connecting throws the wrong error }); } + + if (failureScenario == FailureScenario.TimedOut) + { + return new ExceptionContract( + new[]{ + typeof(HalibutClientException) + }, + new[] + { + "TODO" + }); + } } throw new NotImplementedException(); @@ -229,6 +207,7 @@ public enum FileTransferServiceOperation public enum FailureScenario { ConnectionFaulted, - ScriptExecutionCancelled + ScriptExecutionCancelled, + TimedOut } }