diff --git a/README.md b/README.md index c7ed51f..96ee4d7 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,22 @@ public void Default_Reaches3(int id) ``` The same optional arguments (max retries and delay between each retry) are supported as for facts, and can be used in the same way. +### Skipping tests at Runtime +In addition to retries, `RetryFact` and `RetryTheory` both support dynamically skipping tests at runtime. To make a test skip just use `Skip.Always()` +within your test code. +It also supports custom exception types so you can skip a test if a type of exception gets thrown. You do this by specifying the exception type to the +attribute above your test, e.g. +```cs +[RetryFact(typeof(TestException))] +public void CustomException_SkipsAtRuntime() +{ + throw new TestException(); +} +``` +This functionality also allows for skipping to work when you are already using another library for dynamically skipping tests by specifying the exception +type that is used by that library to the `RetryFact`. e.g. if you are using the popular Xunit.SkippableFact nuget package and want to add retries, converting the +test is as simple as replacing `[SkippableFact]` with `[RetryFact(typeof(Xunit.SkipException))]` above the test and you don't need to change the test itself. + ## Viewing retry logs By default, you won't see whether your tests are being retried as we make this information available via the xunit diagnostic logs but test runners will hide these detailed logs by default. diff --git a/build/Makefile b/build/Makefile index c81134c..736603a 100644 --- a/build/Makefile +++ b/build/Makefile @@ -1,4 +1,4 @@ -VERSION=1.6.0# +VERSION=1.7.0# clean: rm -r ../artefacts || true diff --git a/src/xRetry.SpecFlow/SkipException.cs b/src/xRetry.SpecFlow/SkipException.cs new file mode 100644 index 0000000..68bf5a7 --- /dev/null +++ b/src/xRetry.SpecFlow/SkipException.cs @@ -0,0 +1,14 @@ +using System; + +// ReSharper disable once CheckNamespace +// ReSharper disable once IdentifierTypo +namespace Xunit +{ + /// + /// Do not use. + /// Exists purely as a marker to replicate the exception thrown by Xunit.SkippableFact that SpecFlow.xUnit + /// makes use of. That way we can intercept the exception that is throwing without also having our own runtime + /// plugin, or adding a direct dependency on either of these other libraries. + /// + internal class SkipException : Exception { } +} diff --git a/src/xRetry.SpecFlow/TestGeneratorProvider.cs b/src/xRetry.SpecFlow/TestGeneratorProvider.cs index 25ffd90..7ba7b8b 100644 --- a/src/xRetry.SpecFlow/TestGeneratorProvider.cs +++ b/src/xRetry.SpecFlow/TestGeneratorProvider.cs @@ -59,17 +59,21 @@ public override void SetTestMethodCategories(TestClassGenerationContext generati CodeAttributeDeclaration retryAttribute = CodeDomHelper.AddAttribute(testMethod, "xRetry.Retry" + (originalAttribute.Name == FACT_ATTRIBUTE ? "Fact" : "Theory")); - if (retryTag.MaxRetries != null) - { - retryAttribute.Arguments.Add( - new CodeAttributeArgument(new CodePrimitiveExpression(retryTag.MaxRetries))); - - if(retryTag.DelayBetweenRetriesMs != null) - { - retryAttribute.Arguments.Add( - new CodeAttributeArgument(new CodePrimitiveExpression(retryTag.DelayBetweenRetriesMs))); - } - } + retryAttribute.Arguments.Add(new CodeAttributeArgument( + new CodePrimitiveExpression(retryTag.MaxRetries ?? RetryFactAttribute.DEFAULT_MAX_RETRIES))); + retryAttribute.Arguments.Add(new CodeAttributeArgument( + new CodePrimitiveExpression(retryTag.DelayBetweenRetriesMs ?? + RetryFactAttribute.DEFAULT_DELAY_BETWEEN_RETRIES_MS))); + + // Always skip on Xunit.SkipException (from Xunit.SkippableFact) which is used by SpecFlow.xUnit to implement + // dynamic test skipping. This way we can intercept the exception that is already thrown without also having + // our own runtime plugin. + retryAttribute.Arguments.Add(new CodeAttributeArgument( + new CodeArrayCreateExpression(new CodeTypeReference(typeof(Type)), + new CodeExpression[] + { + new CodeTypeOfExpression(typeof(Xunit.SkipException)) + }))); // Copy arguments from the original attribute for (int i = 0; i < originalAttribute.Arguments.Count; i++) diff --git a/src/xRetry.SpecFlow/xRetry.SpecFlow.csproj b/src/xRetry.SpecFlow/xRetry.SpecFlow.csproj index 51f0bdb..9257d48 100644 --- a/src/xRetry.SpecFlow/xRetry.SpecFlow.csproj +++ b/src/xRetry.SpecFlow/xRetry.SpecFlow.csproj @@ -12,4 +12,8 @@ + + + + diff --git a/src/xRetry/BlockingMessageBus.cs b/src/xRetry/BlockingMessageBus.cs index 829d287..1b7e7d6 100644 --- a/src/xRetry/BlockingMessageBus.cs +++ b/src/xRetry/BlockingMessageBus.cs @@ -10,16 +10,20 @@ namespace xRetry public class BlockingMessageBus : IMessageBus { private readonly IMessageBus underlyingMessageBus; + private readonly MessageTransformer messageTransformer; private ConcurrentQueue messageQueue = new ConcurrentQueue(); - public BlockingMessageBus(IMessageBus underlyingMessageBus) + public BlockingMessageBus(IMessageBus underlyingMessageBus, MessageTransformer messageTransformer) { this.underlyingMessageBus = underlyingMessageBus; + this.messageTransformer = messageTransformer; } - public bool QueueMessage(IMessageSinkMessage message) + public bool QueueMessage(IMessageSinkMessage rawMessage) { - messageQueue.Enqueue(message); + // Transform the message to apply any additional functionality, then intercept & store it for replay later + IMessageSinkMessage transformedMessage = messageTransformer.Transform(rawMessage); + messageQueue.Enqueue(transformedMessage); // Returns if execution should continue. Since we are intercepting the message, we // have no way of checking this so always continue... diff --git a/src/xRetry/Exceptions/SkipTestException.cs b/src/xRetry/Exceptions/SkipTestException.cs new file mode 100644 index 0000000..f923fda --- /dev/null +++ b/src/xRetry/Exceptions/SkipTestException.cs @@ -0,0 +1,30 @@ +using System; +using System.Runtime.Serialization; + +namespace xRetry.Exceptions +{ + [Serializable] + public class SkipTestException : Exception + { + public readonly string Reason; + + public SkipTestException(string reason) + : base("Test skipped. Reason: " + reason) + { + Reason = reason; + } + + protected SkipTestException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + Reason = info.GetString(nameof(Reason)); + } + + public override void GetObjectData(SerializationInfo info, StreamingContext context) + { + info.AddValue(nameof(Reason), Reason); + + base.GetObjectData(info, context); + } + } +} diff --git a/src/xRetry/Extensions/EnumerableExtensions.cs b/src/xRetry/Extensions/EnumerableExtensions.cs new file mode 100644 index 0000000..6d51872 --- /dev/null +++ b/src/xRetry/Extensions/EnumerableExtensions.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace xRetry.Extensions +{ + public static class EnumerableExtensions + { + public static bool ContainsAny(this IEnumerable values, T[] searchFor, IEqualityComparer comparer = null) + { + if (searchFor == null) + { + throw new ArgumentNullException(nameof(searchFor)); + } + if (comparer == null) + { + comparer = EqualityComparer.Default; + } + + return searchFor.Length != 0 && + values.Any(val => searchFor.Any(search => comparer.Equals(val, search))); + } + } +} diff --git a/src/xRetry/IRetryableTestCase.cs b/src/xRetry/IRetryableTestCase.cs index 366486b..7832b8f 100644 --- a/src/xRetry/IRetryableTestCase.cs +++ b/src/xRetry/IRetryableTestCase.cs @@ -6,5 +6,6 @@ public interface IRetryableTestCase : IXunitTestCase { int MaxRetries { get; } int DelayBetweenRetriesMs { get; } + string[] SkipOnExceptionFullNames { get; } } } diff --git a/src/xRetry/MessageTransformer.cs b/src/xRetry/MessageTransformer.cs new file mode 100644 index 0000000..703b0cf --- /dev/null +++ b/src/xRetry/MessageTransformer.cs @@ -0,0 +1,39 @@ +using System.Linq; +using xRetry.Extensions; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace xRetry +{ + public class MessageTransformer + { + private readonly string[] skipOnExceptionFullNames; + + public bool Skipped { get; private set; } + + public MessageTransformer(string[] skipOnExceptionFullNames) + { + this.skipOnExceptionFullNames = skipOnExceptionFullNames; + } + + /// + /// Transforms a message received from an xUnit test into another message, replacing it + /// where necessary to add additional functionality, e.g. dynamic skipping + /// + /// + /// + public IMessageSinkMessage Transform(IMessageSinkMessage message) + { + // If this is a message saying that the test has been skipped, replace the message with skipping the test + if (message is TestFailed failed && failed.ExceptionTypes.ContainsAny(skipOnExceptionFullNames)) + { + string reason = failed.Messages?.FirstOrDefault(); + Skipped = true; + return new TestSkipped(failed.Test, reason); + } + + // Otherwise this isn't a message saying the test is skipped, follow usual intercept for replay later behaviour + return message; + } + } +} diff --git a/src/xRetry/RetryFactAttribute.cs b/src/xRetry/RetryFactAttribute.cs index 6819e30..fa15f8d 100644 --- a/src/xRetry/RetryFactAttribute.cs +++ b/src/xRetry/RetryFactAttribute.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using Xunit; using Xunit.Sdk; @@ -12,15 +13,38 @@ namespace xRetry [AttributeUsage(AttributeTargets.Method)] public class RetryFactAttribute : FactAttribute { - public readonly int MaxRetries; - public readonly int DelayBetweenRetriesMs; + public const int DEFAULT_MAX_RETRIES = 3; + public const int DEFAULT_DELAY_BETWEEN_RETRIES_MS = 0; + + public readonly int MaxRetries = DEFAULT_MAX_RETRIES; + public readonly int DelayBetweenRetriesMs = DEFAULT_DELAY_BETWEEN_RETRIES_MS; + public readonly Type[] SkipOnExceptions; + + /// + /// Ctor (just skip on exceptions) + /// + /// Mark the test as skipped when this type of exception is encountered + public RetryFactAttribute(params Type[] skipOnExceptions) + { + SkipOnExceptions = skipOnExceptions ?? Type.EmptyTypes; + + if (SkipOnExceptions.Any(t => !t.IsSubclassOf(typeof(Exception)))) + { + throw new ArgumentException("Specified type must be an exception", nameof(skipOnExceptions)); + } + } /// - /// Ctor + /// Ctor (full) /// /// The number of times to run a test for until it succeeds /// The amount of time (in ms) to wait between each test run attempt - public RetryFactAttribute(int maxRetries = 3, int delayBetweenRetriesMs = 0) + /// Mark the test as skipped when this type of exception is encountered + public RetryFactAttribute( + int maxRetries = DEFAULT_MAX_RETRIES, + int delayBetweenRetriesMs = DEFAULT_DELAY_BETWEEN_RETRIES_MS, + params Type[] skipOnExceptions) + : this(skipOnExceptions) { if (maxRetries < 1) { diff --git a/src/xRetry/RetryFactDiscoverer.cs b/src/xRetry/RetryFactDiscoverer.cs index 1884c91..67b6966 100644 --- a/src/xRetry/RetryFactDiscoverer.cs +++ b/src/xRetry/RetryFactDiscoverer.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.Linq; using Xunit.Abstractions; @@ -36,8 +37,12 @@ public IEnumerable Discover(ITestFrameworkDiscoveryOptions disco int maxRetries = factAttribute.GetNamedArgument(nameof(RetryFactAttribute.MaxRetries)); int delayBetweenRetriesMs = factAttribute.GetNamedArgument(nameof(RetryFactAttribute.DelayBetweenRetriesMs)); + Type[] skipOnExceptions = + factAttribute.GetNamedArgument(nameof(RetryTheoryAttribute.SkipOnExceptions)); + testCase = new RetryTestCase(messageSink, discoveryOptions.MethodDisplayOrDefault(), - discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, maxRetries, delayBetweenRetriesMs); + discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, maxRetries, delayBetweenRetriesMs, + skipOnExceptions); } return new[] { testCase }; diff --git a/src/xRetry/RetryTestCase.cs b/src/xRetry/RetryTestCase.cs index ff0b91a..724601b 100644 --- a/src/xRetry/RetryTestCase.cs +++ b/src/xRetry/RetryTestCase.cs @@ -2,6 +2,7 @@ using System.ComponentModel; using System.Threading; using System.Threading.Tasks; +using xRetry.Exceptions; using Xunit.Abstractions; using Xunit.Sdk; @@ -12,6 +13,7 @@ public class RetryTestCase : XunitTestCase, IRetryableTestCase { public int MaxRetries { get; private set; } public int DelayBetweenRetriesMs { get; private set; } + public string[] SkipOnExceptionFullNames { get; private set; } [EditorBrowsable(EditorBrowsableState.Never)] [Obsolete( @@ -25,12 +27,14 @@ public RetryTestCase( ITestMethod testMethod, int maxRetries, int delayBetweenRetriesMs, + Type[] skipOnExceptions, object[] testMethodArguments = null) : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod, testMethodArguments) { MaxRetries = maxRetries; DelayBetweenRetriesMs = delayBetweenRetriesMs; + SkipOnExceptionFullNames = GetSkipOnExceptionFullNames(skipOnExceptions); } public override Task RunAsync(IMessageSink diagnosticMessageSink, IMessageBus messageBus, @@ -47,6 +51,7 @@ public override void Serialize(IXunitSerializationInfo data) data.AddValue("MaxRetries", MaxRetries); data.AddValue("DelayBetweenRetriesMs", DelayBetweenRetriesMs); + data.AddValue("SkipOnExceptionFullNames", SkipOnExceptionFullNames); } public override void Deserialize(IXunitSerializationInfo data) @@ -55,6 +60,18 @@ public override void Deserialize(IXunitSerializationInfo data) MaxRetries = data.GetValue("MaxRetries"); DelayBetweenRetriesMs = data.GetValue("DelayBetweenRetriesMs"); + SkipOnExceptionFullNames = data.GetValue("SkipOnExceptionFullNames"); + } + + public static string[] GetSkipOnExceptionFullNames(Type[] customSkipOnExceptions) + { + string[] toRet = new string[customSkipOnExceptions.Length + 1]; + for (int i = 0; i < customSkipOnExceptions.Length; i++) + { + toRet[i] = customSkipOnExceptions[i].FullName; + } + toRet[toRet.Length - 1] = typeof(SkipTestException).FullName; + return toRet; } } } diff --git a/src/xRetry/RetryTestCaseRunner.cs b/src/xRetry/RetryTestCaseRunner.cs index 58ffe89..759e027 100644 --- a/src/xRetry/RetryTestCaseRunner.cs +++ b/src/xRetry/RetryTestCaseRunner.cs @@ -28,14 +28,21 @@ public static async Task RunAsync( { // Prevent messages from the test run from being passed through, as we don't want // a message to mark the test as failed when we're going to retry it - using (BlockingMessageBus blockingMessageBus = new BlockingMessageBus(messageBus)) + MessageTransformer messageTransformer = new MessageTransformer(testCase.SkipOnExceptionFullNames); + using (BlockingMessageBus blockingMessageBus = new BlockingMessageBus(messageBus, messageTransformer)) { diagnosticMessageSink.OnMessage(new DiagnosticMessage("Running test \"{0}\" attempt ({1}/{2})", testCase.DisplayName, i, testCase.MaxRetries)); RunSummary summary = await fnRunSingle(blockingMessageBus); - // If we succeeded, or we've reached the max retries return the result + if (messageTransformer.Skipped) + { + summary.Failed = 0; + summary.Skipped = 1; + } + + // If we succeeded, skipped, or we've reached the max retries return the result if (summary.Failed == 0 || i == testCase.MaxRetries) { // If we have failed (after all retries, log that) diff --git a/src/xRetry/RetryTheoryAttribute.cs b/src/xRetry/RetryTheoryAttribute.cs index f33c27c..0b24c65 100644 --- a/src/xRetry/RetryTheoryAttribute.cs +++ b/src/xRetry/RetryTheoryAttribute.cs @@ -12,7 +12,14 @@ namespace xRetry public class RetryTheoryAttribute : RetryFactAttribute { /// - public RetryTheoryAttribute(int maxRetries = 3, int delayBetweenRetriesMs = 0) - : base(maxRetries, delayBetweenRetriesMs) { } + public RetryTheoryAttribute(params Type[] skipOnExceptions) + : base(skipOnExceptions) { } + + /// + public RetryTheoryAttribute( + int maxRetries = DEFAULT_MAX_RETRIES, + int delayBetweenRetriesMs = DEFAULT_DELAY_BETWEEN_RETRIES_MS, + params Type[] skipOnExceptions) + : base(maxRetries, delayBetweenRetriesMs, skipOnExceptions) { } } } diff --git a/src/xRetry/RetryTheoryDiscoverer.cs b/src/xRetry/RetryTheoryDiscoverer.cs index f0aa946..3b7e701 100644 --- a/src/xRetry/RetryTheoryDiscoverer.cs +++ b/src/xRetry/RetryTheoryDiscoverer.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using Xunit.Abstractions; using Xunit.Sdk; @@ -18,6 +19,8 @@ protected override IEnumerable CreateTestCasesForDataRow( int maxRetries = theoryAttribute.GetNamedArgument(nameof(RetryTheoryAttribute.MaxRetries)); int delayBetweenRetriesMs = theoryAttribute.GetNamedArgument(nameof(RetryTheoryAttribute.DelayBetweenRetriesMs)); + Type[] skipOnExceptions = + theoryAttribute.GetNamedArgument(nameof(RetryTheoryAttribute.SkipOnExceptions)); return new[] { new RetryTestCase( @@ -27,6 +30,7 @@ protected override IEnumerable CreateTestCasesForDataRow( testMethod, maxRetries, delayBetweenRetriesMs, + skipOnExceptions, dataRow) }; } @@ -37,11 +41,13 @@ protected override IEnumerable CreateTestCasesForTheory( int maxRetries = theoryAttribute.GetNamedArgument(nameof(RetryTheoryAttribute.MaxRetries)); int delayBetweenRetriesMs = theoryAttribute.GetNamedArgument(nameof(RetryTheoryAttribute.DelayBetweenRetriesMs)); + Type[] skipOnExceptions = + theoryAttribute.GetNamedArgument(nameof(RetryTheoryAttribute.SkipOnExceptions)); return new[] { new RetryTheoryDiscoveryAtRuntimeCase(DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), - discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, maxRetries, delayBetweenRetriesMs) + discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, maxRetries, delayBetweenRetriesMs, skipOnExceptions) }; } } diff --git a/src/xRetry/RetryTheoryDiscoveryAtRuntimeCase.cs b/src/xRetry/RetryTheoryDiscoveryAtRuntimeCase.cs index ed01d29..80d6ca7 100644 --- a/src/xRetry/RetryTheoryDiscoveryAtRuntimeCase.cs +++ b/src/xRetry/RetryTheoryDiscoveryAtRuntimeCase.cs @@ -17,6 +17,7 @@ public class RetryTheoryDiscoveryAtRuntimeCase : XunitTestCase, IRetryableTestCa { public int MaxRetries { get; private set; } public int DelayBetweenRetriesMs { get; private set; } + public string[] SkipOnExceptionFullNames { get; private set; } /// [EditorBrowsable(EditorBrowsableState.Never)] @@ -29,11 +30,13 @@ public RetryTheoryDiscoveryAtRuntimeCase( TestMethodDisplayOptions defaultMethodDisplayOptions, ITestMethod testMethod, int maxRetries, - int delayBetweenRetriesMs) + int delayBetweenRetriesMs, + Type[] skipOnExceptions) : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod) { MaxRetries = maxRetries; DelayBetweenRetriesMs = delayBetweenRetriesMs; + SkipOnExceptionFullNames = RetryTestCase.GetSkipOnExceptionFullNames(skipOnExceptions); } /// @@ -51,6 +54,7 @@ public override void Serialize(IXunitSerializationInfo data) data.AddValue("MaxRetries", MaxRetries); data.AddValue("DelayBetweenRetriesMs", DelayBetweenRetriesMs); + data.AddValue("SkipOnExceptionFullNames", SkipOnExceptionFullNames); } public override void Deserialize(IXunitSerializationInfo data) @@ -59,6 +63,7 @@ public override void Deserialize(IXunitSerializationInfo data) MaxRetries = data.GetValue("MaxRetries"); DelayBetweenRetriesMs = data.GetValue("DelayBetweenRetriesMs"); + SkipOnExceptionFullNames = data.GetValue("SkipOnExceptionFullNames"); } } } diff --git a/src/xRetry/Skip.cs b/src/xRetry/Skip.cs new file mode 100644 index 0000000..b24e5f9 --- /dev/null +++ b/src/xRetry/Skip.cs @@ -0,0 +1,16 @@ +using xRetry.Exceptions; + +namespace xRetry +{ + public static class Skip + { + /// + /// Throws an exception that results in a "Skipped" result for the test. + /// + /// Reason for the test needing to be skipped + public static void Always(string reason = null) + { + throw new SkipTestException(reason); + } + } +} diff --git a/test/UnitTests/Exceptions/SkipTestExceptionTests.cs b/test/UnitTests/Exceptions/SkipTestExceptionTests.cs new file mode 100644 index 0000000..c7d4f91 --- /dev/null +++ b/test/UnitTests/Exceptions/SkipTestExceptionTests.cs @@ -0,0 +1,37 @@ +using System.IO; +using System.Runtime.Serialization.Formatters.Binary; +using AutoFixture; +using FluentAssertions; +using xRetry.Exceptions; +using Xunit; + +namespace UnitTests.Exceptions +{ + public class SkipTestExceptionTests + { + [Fact] + public void Serialisation_RoundTrip_RetainsData() + { + // Arrange + Fixture fixture = new Fixture(); + SkipTestException expected = new SkipTestException(fixture.Create()); + + // Act + SkipTestException actual; + BinaryFormatter formatter = new BinaryFormatter(); + using (MemoryStream s = new MemoryStream()) + { +#pragma warning disable SYSLIB0011 // Type or member is obsolete +#pragma warning disable 618 + formatter.Serialize(s, expected); + s.Position = 0; + actual = (SkipTestException) formatter.Deserialize(s); +#pragma warning restore SYSLIB0011 // Type or member is obsolete +#pragma warning restore 618 + } + + // Assert + actual.Should().BeEquivalentTo(expected); + } + } +} diff --git a/test/UnitTests/Facts/RetryFactRuntimeSkipTests.cs b/test/UnitTests/Facts/RetryFactRuntimeSkipTests.cs new file mode 100644 index 0000000..1c052e7 --- /dev/null +++ b/test/UnitTests/Facts/RetryFactRuntimeSkipTests.cs @@ -0,0 +1,38 @@ +using UnitTests.TestClasses; +using xRetry; +using Xunit; +using Skip = xRetry.Skip; + +namespace UnitTests.Facts +{ + public class RetryFactRuntimeSkipTests + { + [RetryFact] + public void SkipAtRuntime() + { + // Note: All we're doing with this test is checking that the rest of the test doesn't get run + // checking it's skipped (and doesn't pass) would need to be done manually. + Skip.Always(); + + Assert.True(false, "Should have been skipped . . ."); + } + + [RetryFact(typeof(TestException))] + public void CustomException_SkipsAtRuntime() + { + throw new TestException(); + } + + private static int skippedNumCalls = 0; + + [RetryFact] + public void Skip_DoesNotRetry() + { + // Assertion would fail on subsequent attempts, before reaching the skip + skippedNumCalls++; + Assert.Equal(1, skippedNumCalls); + + Skip.Always(); + } + } +} diff --git a/test/UnitTests/RetryFactAttributeTests.cs b/test/UnitTests/RetryFactAttributeTests.cs new file mode 100644 index 0000000..0d59796 --- /dev/null +++ b/test/UnitTests/RetryFactAttributeTests.cs @@ -0,0 +1,60 @@ +using System; +using AutoFixture; +using FluentAssertions; +using xRetry; +using Xunit; + +namespace UnitTests +{ + public class RetryFactAttributeTests + { + [Fact] + public void Ctor_Empty_NoSkipOnExceptions() + { + // Arrange & Act + RetryFactAttribute attr = new RetryFactAttribute(); + + // Assert + attr.SkipOnExceptions.Should().BeEmpty(); + } + + [Fact] + public void SkipOnExceptionsCtor_Exceptions_ShouldSave() + { + // Arrange + Type[] expected = new[] + { + typeof(ArgumentException), + typeof(ArgumentNullException) + }; + + // Act + RetryFactAttribute attr = new RetryFactAttribute(expected); + + // Assert + attr.SkipOnExceptions.Should().BeEquivalentTo(expected); + } + + [Fact] + public void FullCtr_Exceptions_ShouldSave() + { + // Arrange + Fixture fixture = new Fixture(); + Type[] expected = new[] + { + typeof(ArgumentException), + typeof(ArgumentNullException) + }; + + // Act + RetryFactAttribute attr = new RetryFactAttribute(fixture.Create(), fixture.Create(), expected); + + // Assert + attr.SkipOnExceptions.Should().BeEquivalentTo(expected); + } + + [Fact] + public void Ctor_NonExceptionTypes_ShouldThrow() => + Assert.Throws(() => new RetryFactAttribute(typeof(RetryFactAttributeTests))); + } +} diff --git a/test/UnitTests/SpecFlow/Features/RetryRuntimeIgnoreScenarios.feature b/test/UnitTests/SpecFlow/Features/RetryRuntimeIgnoreScenarios.feature new file mode 100644 index 0000000..75e4dc7 --- /dev/null +++ b/test/UnitTests/SpecFlow/Features/RetryRuntimeIgnoreScenarios.feature @@ -0,0 +1,19 @@ +Feature: Retry Ignore Scenarios + In order to allow for tests to be ignored/skipped at runtime + So that the full feature set of SpecFlow is still available with xRetry (IUnitTestRuntimeProvider.TestIgnore) + As a QA engineer + I want to be able to ignore/skip tests + +@retry +Scenario: Test is ignored at runtime + When I ignore this test + Then fail because this test should have been skipped + +@retry +Scenario Outline: Test (outline) is ignored at runtime + When I ignore this test + Then fail because this test should have been skipped + Examples: + | n | + | 1 | + | 2 | \ No newline at end of file diff --git a/test/UnitTests/SpecFlow/Steps/RuntimeIgnoreSteps.cs b/test/UnitTests/SpecFlow/Steps/RuntimeIgnoreSteps.cs new file mode 100644 index 0000000..8e47d36 --- /dev/null +++ b/test/UnitTests/SpecFlow/Steps/RuntimeIgnoreSteps.cs @@ -0,0 +1,22 @@ +using TechTalk.SpecFlow; +using TechTalk.SpecFlow.UnitTestProvider; + +namespace UnitTests.SpecFlow.Steps +{ + [Binding] + public class RuntimeIgnoreSteps + { + private readonly IUnitTestRuntimeProvider unitTestRuntimeProvider; + + public RuntimeIgnoreSteps(IUnitTestRuntimeProvider unitTestRuntimeProvider) + { + this.unitTestRuntimeProvider = unitTestRuntimeProvider; + } + + [When(@"I ignore this test")] + public void WhenIIgnoreThisTest() + { + unitTestRuntimeProvider.TestIgnore("Ignored at runtime"); + } + } +} diff --git a/test/UnitTests/TestClasses/NonSerializableTestData.cs b/test/UnitTests/TestClasses/NonSerializableTestData.cs new file mode 100644 index 0000000..fc66036 --- /dev/null +++ b/test/UnitTests/TestClasses/NonSerializableTestData.cs @@ -0,0 +1,12 @@ +namespace UnitTests.TestClasses +{ + public class NonSerializableTestData + { + public readonly int Id; + + public NonSerializableTestData(int id) + { + Id = id; + } + } +} diff --git a/test/UnitTests/TestClasses/TestException.cs b/test/UnitTests/TestClasses/TestException.cs new file mode 100644 index 0000000..683f8d8 --- /dev/null +++ b/test/UnitTests/TestClasses/TestException.cs @@ -0,0 +1,6 @@ +using System; + +namespace UnitTests.TestClasses +{ + public class TestException : Exception { } +} diff --git a/test/UnitTests/Theories/RetryTheoryNonSerializableDataTests.cs b/test/UnitTests/Theories/RetryTheoryNonSerializableDataTests.cs index 727aaa1..4ff5b2b 100644 --- a/test/UnitTests/Theories/RetryTheoryNonSerializableDataTests.cs +++ b/test/UnitTests/Theories/RetryTheoryNonSerializableDataTests.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using UnitTests.TestClasses; using xRetry; using Xunit; @@ -32,15 +33,5 @@ public static IEnumerable GetTestData() => new[] new object[] { new NonSerializableTestData(0) }, new object[] { new NonSerializableTestData(1) } }; - - public class NonSerializableTestData - { - public readonly int Id; - - public NonSerializableTestData(int id) - { - Id = id; - } - } } } diff --git a/test/UnitTests/Theories/RetryTheoryRuntimeSkipNonSerializableDataTests.cs b/test/UnitTests/Theories/RetryTheoryRuntimeSkipNonSerializableDataTests.cs new file mode 100644 index 0000000..39717c4 --- /dev/null +++ b/test/UnitTests/Theories/RetryTheoryRuntimeSkipNonSerializableDataTests.cs @@ -0,0 +1,53 @@ +using System.Collections.Generic; +using UnitTests.TestClasses; +using xRetry; +using Xunit; +using Skip = xRetry.Skip; + +namespace UnitTests.Theories +{ + public class RetryTheoryRuntimeSkipNonSerializableDataTests + { + [RetryTheory] + [MemberData(nameof(GetTestData))] + public void SkipAtRuntime(NonSerializableTestData _) + { + // Note: All we're doing with this test is checking that the rest of the test doesn't get run + // checking it's skipped (and doesn't pass) would need to be done manually. + Skip.Always(); + + Assert.True(false, "Should have been skipped . . ."); + } + + [RetryTheory(typeof(TestException))] + [MemberData(nameof(GetTestData))] + public void CustomException_SkipsAtRuntime(NonSerializableTestData _) + { + throw new TestException(); + } + + // testId => numCalls + private static readonly Dictionary skippedNumCalls = new Dictionary() + { + { 0, 0 }, + { 1, 0 } + }; + + [RetryTheory] + [MemberData(nameof(GetTestData))] + public void Skip_DoesNotRetry(NonSerializableTestData nonSerializableWrapper) + { + // Assertion would fail on subsequent attempts, before reaching the skip + skippedNumCalls[nonSerializableWrapper.Id]++; + Assert.Equal(1, skippedNumCalls[nonSerializableWrapper.Id]); + + Skip.Always(); + } + + public static IEnumerable GetTestData() => new[] + { + new object[] { new NonSerializableTestData(0) }, + new object[] { new NonSerializableTestData(1) } + }; + } +} diff --git a/test/UnitTests/Theories/RetryTheoryRuntimeSkipTests.cs b/test/UnitTests/Theories/RetryTheoryRuntimeSkipTests.cs new file mode 100644 index 0000000..5913007 --- /dev/null +++ b/test/UnitTests/Theories/RetryTheoryRuntimeSkipTests.cs @@ -0,0 +1,50 @@ +using System.Collections.Generic; +using UnitTests.TestClasses; +using xRetry; +using Xunit; +using Skip = xRetry.Skip; + +namespace UnitTests.Theories +{ + public class RetryTheoryRuntimeSkipTests + { + [RetryTheory] + [InlineData(0)] + [InlineData(1)] + public void SkipAtRuntime(int _) + { + // Note: All we're doing with this test is checking that the rest of the test doesn't get run + // checking it's skipped (and doesn't pass) would need to be done manually. + Skip.Always(); + + Assert.True(false, "Should have been skipped . . ."); + } + + [RetryTheory(typeof(TestException))] + [InlineData(0)] + [InlineData(1)] + public void CustomException_SkipsAtRuntime(int _) + { + throw new TestException(); + } + + // testId => numCalls + private static readonly Dictionary skippedNumCalls = new Dictionary() + { + { 0, 0 }, + { 1, 0 } + }; + + [RetryTheory] + [InlineData(0)] + [InlineData(1)] + public void Skip_DoesNotRetry(int id) + { + // Assertion would fail on subsequent attempts, before reaching the skip + skippedNumCalls[id]++; + Assert.Equal(1, skippedNumCalls[id]); + + Skip.Always(); + } + } +} diff --git a/test/UnitTests/UnitTests.csproj b/test/UnitTests/UnitTests.csproj index a08884c..d963717 100644 --- a/test/UnitTests/UnitTests.csproj +++ b/test/UnitTests/UnitTests.csproj @@ -17,6 +17,8 @@ + +