From 3f4a29774639f64b432e4faecbe5c11f979e3b47 Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Mon, 30 Sep 2024 14:50:53 +1300 Subject: [PATCH] Ignore more tests that rely on timing. Enable tests that usually fail due to disk space issues --- .../ClientFileTransferRetriesTimeout.cs | 2 ++ ...ntFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs | 2 ++ ...ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs | 2 ++ .../ClientScriptExecutionRetriesTimeout.cs | 2 +- ...criptExecutorObservesScriptObserverBackoffStrategy.cs | 2 ++ .../FileTransferServiceTests.cs | 2 -- .../ScriptServiceV2IntegrationTest.cs | 2 +- ...pOnEnvironmentsWithKnownPerformanceIssuesAttribute.cs | 9 ++++++++- .../TentacleCommandLineTests.cs | 2 +- .../Util/SilentProcessRunnerFixture.cs | 8 +++++++- 10 files changed, 26 insertions(+), 7 deletions(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs index d2e5784d1..f33ba0bf0 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs @@ -11,6 +11,7 @@ using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; using Octopus.Tentacle.Tests.Integration.Support; using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; using Octopus.Tentacle.Tests.Integration.Util; using Octopus.Tentacle.Tests.Integration.Util.Builders; using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; @@ -24,6 +25,7 @@ namespace Octopus.Tentacle.Tests.Integration /// from RPC calls when they are being retried and the rpc timeout period elapses. /// [IntegrationTestTimeout] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public class ClientFileTransferRetriesTimeout : IntegrationTest { readonly TimeSpan retryIfRemainingDurationAtLeastBuffer = TimeSpan.FromSeconds(1); diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs index b8ebd1261..cb5876c7f 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs @@ -7,6 +7,7 @@ using Octopus.Tentacle.Contracts.ClientServices; using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; using Octopus.Tentacle.Tests.Integration.Util.Builders; using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; @@ -14,6 +15,7 @@ namespace Octopus.Tentacle.Tests.Integration { [IntegrationTestTimeout] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public class ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled : IntegrationTest { [Test] diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs index df282f0fd..eef4f972f 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs @@ -10,6 +10,7 @@ using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; using Octopus.Tentacle.Tests.Integration.Support; using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; using Octopus.Tentacle.Tests.Integration.Util.Builders; using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; @@ -20,6 +21,7 @@ public class ClientFileTransfersAreRetriedWhenRetriesAreEnabled : IntegrationTes { [Test] [TentacleConfigurations(testCommonVersions: true, scriptServiceToTest: ScriptServiceVersionToTest.None)] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public async Task FailedUploadsAreRetriedAndIsEventuallySuccessful(TentacleConfigurationTestCase tentacleConfigurationTestCase) { await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionRetriesTimeout.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionRetriesTimeout.cs index 6ff609a88..2ea063101 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionRetriesTimeout.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionRetriesTimeout.cs @@ -218,7 +218,7 @@ public async Task WhenRpcRetriesTimeOut_DuringStartScript_TheRpcCallIsCancelled( [Test] [TentacleConfigurations] - [SkipOnEnvironmentsWithKnownPerformanceIssues] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public async Task WhenStartScriptFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) { var retryDuration = TimeSpan.FromSeconds(15); diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutorObservesScriptObserverBackoffStrategy.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutorObservesScriptObserverBackoffStrategy.cs index b61e5aeb0..949d586ae 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutorObservesScriptObserverBackoffStrategy.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutorObservesScriptObserverBackoffStrategy.cs @@ -6,6 +6,7 @@ using Octopus.Tentacle.Contracts.ClientServices; using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; using Octopus.Tentacle.Tests.Integration.Util; using Octopus.Tentacle.Tests.Integration.Util.Builders; using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; @@ -13,6 +14,7 @@ namespace Octopus.Tentacle.Tests.Integration { [IntegrationTestTimeout] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public class ClientScriptExecutorObservesScriptObserverBackoffStrategy : IntegrationTest { [Test] diff --git a/source/Octopus.Tentacle.Tests.Integration/FileTransferServiceTests.cs b/source/Octopus.Tentacle.Tests.Integration/FileTransferServiceTests.cs index 8d3bfb513..dd166d8bc 100644 --- a/source/Octopus.Tentacle.Tests.Integration/FileTransferServiceTests.cs +++ b/source/Octopus.Tentacle.Tests.Integration/FileTransferServiceTests.cs @@ -16,7 +16,6 @@ public class FileTransferServiceTests : IntegrationTest { [Test] [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - [SkipOnEnvironmentsWithKnownPerformanceIssues] public async Task UploadFileSuccessfully(TentacleConfigurationTestCase tentacleConfigurationTestCase) { using var fileToUpload = new RandomTemporaryFileBuilder().Build(); @@ -44,7 +43,6 @@ public async Task UploadFileSuccessfully(TentacleConfigurationTestCase tentacleC [Test] [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - [SkipOnEnvironmentsWithKnownPerformanceIssues] public async Task DownloadFileSuccessfully(TentacleConfigurationTestCase tentacleConfigurationTestCase) { using var fileToDownload = new RandomTemporaryFileBuilder().Build(); diff --git a/source/Octopus.Tentacle.Tests.Integration/ScriptServiceV2IntegrationTest.cs b/source/Octopus.Tentacle.Tests.Integration/ScriptServiceV2IntegrationTest.cs index d24f90e6e..816318950 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ScriptServiceV2IntegrationTest.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ScriptServiceV2IntegrationTest.cs @@ -133,7 +133,7 @@ await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, Cancellati [Test] [TentacleConfigurations] - [SkipOnEnvironmentsWithKnownPerformanceIssues] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public async Task WhenALongRunningScriptIsCancelled_TheScriptShouldStop(TentacleConfigurationTestCase tentacleConfigurationTestCase) { await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() diff --git a/source/Octopus.Tentacle.Tests.Integration/Support/TestAttributes/SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute.cs b/source/Octopus.Tentacle.Tests.Integration/Support/TestAttributes/SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute.cs index 096322039..493e6cbbc 100644 --- a/source/Octopus.Tentacle.Tests.Integration/Support/TestAttributes/SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute.cs +++ b/source/Octopus.Tentacle.Tests.Integration/Support/TestAttributes/SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute.cs @@ -10,6 +10,13 @@ namespace Octopus.Tentacle.Tests.Integration.Support.TestAttributes [AttributeUsage(AttributeTargets.Class | AttributeTargets.Method)] public class SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute : NUnitAttribute, IApplyToTest { + string Reason { get; } + + public SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute(string reason) + { + Reason = reason; + } + public void ApplyToTest(Test test) { if (test.RunState == RunState.NotRunnable || test.RunState == RunState.Ignored) @@ -18,7 +25,7 @@ public void ApplyToTest(Test test) if (bool.TryParse(Environment.GetEnvironmentVariable("Has_Known_Performance_Issues"), out _)) { test.RunState = RunState.Skipped; - test.Properties.Add("_SKIPREASON", "This test only runs on environments without performance issues"); + test.Properties.Add("_SKIPREASON", $"This test only runs on environments without performance issues because {Reason}"); } } } diff --git a/source/Octopus.Tentacle.Tests.Integration/TentacleCommandLineTests.cs b/source/Octopus.Tentacle.Tests.Integration/TentacleCommandLineTests.cs index f0fc20bbf..42adb51d8 100644 --- a/source/Octopus.Tentacle.Tests.Integration/TentacleCommandLineTests.cs +++ b/source/Octopus.Tentacle.Tests.Integration/TentacleCommandLineTests.cs @@ -28,6 +28,7 @@ namespace Octopus.Tentacle.Tests.Integration /// Please review any changes to the assertions made by these tests carefully. /// [IntegrationTestTimeout] + [SkipOnEnvironmentsWithKnownPerformanceIssues("we don't expect customers to execute it in this environment in a situation where it needs to behave within a reasonable timeframe.")] public class TentacleCommandLineTests : IntegrationTest { [Test] @@ -272,7 +273,6 @@ public async Task CommandSpecificHelpAsJsonLooksSensibleToHumans(TentacleConfigu [Test] [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - [SkipOnEnvironmentsWithKnownPerformanceIssues] public async Task HelpForInstanceSpecificCommandsAlwaysWorks(TentacleConfigurationTestCase tc) { var (_, stdout, stderr) = await RunCommand(tc, null, "help", "--format=json"); diff --git a/source/Octopus.Tentacle.Tests.Integration/Util/SilentProcessRunnerFixture.cs b/source/Octopus.Tentacle.Tests.Integration/Util/SilentProcessRunnerFixture.cs index a7a9983dd..4dd44b0a9 100644 --- a/source/Octopus.Tentacle.Tests.Integration/Util/SilentProcessRunnerFixture.cs +++ b/source/Octopus.Tentacle.Tests.Integration/Util/SilentProcessRunnerFixture.cs @@ -33,6 +33,7 @@ public void SetUpLocal() } [Test] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public void ExitCode_ShouldBeReturned() { using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10))) @@ -56,6 +57,7 @@ public void ExitCode_ShouldBeReturned() } [Test] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public void DebugLogging_ShouldContainDiagnosticsInfo_ForDefault() { using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10))) @@ -83,7 +85,7 @@ public void DebugLogging_ShouldContainDiagnosticsInfo_ForDefault() [Test] [Retry(3)] - [SkipOnEnvironmentsWithKnownPerformanceIssues] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public void CancellationToken_ShouldForceKillTheProcess() { // Terminate the process after a very short time so the test doesn't run forever @@ -116,6 +118,7 @@ public void CancellationToken_ShouldForceKillTheProcess() } [Test] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public void EchoHello_ShouldWriteToStdOut() { using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10))) @@ -138,6 +141,7 @@ public void EchoHello_ShouldWriteToStdOut() } [Test] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public void EchoError_ShouldWriteToStdErr() { using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10))) @@ -160,6 +164,7 @@ public void EchoError_ShouldWriteToStdErr() } [Test] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public void RunAsCurrentUser_ShouldWork() { using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10))) @@ -186,6 +191,7 @@ public void RunAsCurrentUser_ShouldWork() [Test] [WindowsTest] [TestCase("powershell.exe", "-command \"Write-Host $env:userdomain\\$env:username\"")] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public void RunAsCurrentUser_PowerShell_ShouldWork(string command, string arguments) { using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)))