From 0f93e760690dc0043446baee473644ac01824249 Mon Sep 17 00:00:00 2001 From: cliviu Date: Fri, 6 Jan 2017 00:36:01 +0100 Subject: [PATCH 1/3] support for test.retry.count --- .../runtime/junit/SerenityExamplesRunner.java | 59 +++++++++ .../runtime/junit/SerenityFeatureRunner.java | 121 ++++++++++++++++++ .../junit/SerenityScenarioOutlineRunner.java | 34 +++++ .../cucumber/CucumberWithSerenity.java | 104 ++++++++++++++- .../cucumber/SerenityReporter.java | 40 ++++-- .../WhenCreatingSerenityTestOutcomes.groovy | 72 +++++++++++ .../integration/InstableScenario.java | 12 ++ .../integration/steps/SampleWidgetSteps.java | 11 +- .../steps/thucydides/WidgetRetrySteps.java | 17 +++ .../steps/thucydides/WidgetSteps.java | 2 - .../samples/instable_scenario.feature | 8 ++ 11 files changed, 462 insertions(+), 18 deletions(-) create mode 100644 src/main/java/cucumber/runtime/junit/SerenityExamplesRunner.java create mode 100644 src/main/java/cucumber/runtime/junit/SerenityFeatureRunner.java create mode 100644 src/main/java/cucumber/runtime/junit/SerenityScenarioOutlineRunner.java create mode 100644 src/test/java/net/serenitybdd/cucumber/integration/InstableScenario.java create mode 100644 src/test/java/net/serenitybdd/cucumber/integration/steps/thucydides/WidgetRetrySteps.java create mode 100644 src/test/resources/samples/instable_scenario.feature diff --git a/src/main/java/cucumber/runtime/junit/SerenityExamplesRunner.java b/src/main/java/cucumber/runtime/junit/SerenityExamplesRunner.java new file mode 100644 index 0000000..305952e --- /dev/null +++ b/src/main/java/cucumber/runtime/junit/SerenityExamplesRunner.java @@ -0,0 +1,59 @@ +package cucumber.runtime.junit; + +import cucumber.runtime.Runtime; +import cucumber.runtime.model.CucumberExamples; +import cucumber.runtime.model.CucumberScenario; +import org.junit.runner.Runner; +import org.junit.runner.notification.RunNotifier; +import org.junit.runners.Suite; +import org.junit.runners.model.InitializationError; + +import java.util.ArrayList; +import java.util.List; + +public class SerenityExamplesRunner extends Suite { + + private int retryCount; + private Runtime runtime; + + protected SerenityExamplesRunner( + Runtime runtime, + CucumberExamples cucumberExamplesValue, + JUnitReporter jUnitReporter, + int retryCount) throws InitializationError { + super(SerenityExamplesRunner.class, + buildRunners(runtime, cucumberExamplesValue, jUnitReporter)); + this.runtime = runtime; + this.retryCount = retryCount; + } + + private static List buildRunners( + Runtime runtime, + CucumberExamples cucumberExamples, + JUnitReporter jUnitReporter) { + List runners = new ArrayList<>(); + List exampleScenarios = cucumberExamples.createExampleScenarios(); + for (CucumberScenario scenario : exampleScenarios) { + try { + ExecutionUnitRunner exampleScenarioRunner + = new ExecutionUnitRunner(runtime, scenario, jUnitReporter); + runners.add(exampleScenarioRunner); + } catch (InitializationError initializationError) { + initializationError.printStackTrace(); + } + } + return runners; + } + + @Override + protected void runChild(Runner runner, RunNotifier notifier) { + for (int i = 0; i <= this.retryCount; i++) { + runner.run(notifier); + if(runtime.exitStatus() == 0) { + break; + } else { + runtime.getErrors().clear(); + } + } + } +} diff --git a/src/main/java/cucumber/runtime/junit/SerenityFeatureRunner.java b/src/main/java/cucumber/runtime/junit/SerenityFeatureRunner.java new file mode 100644 index 0000000..c65d2a4 --- /dev/null +++ b/src/main/java/cucumber/runtime/junit/SerenityFeatureRunner.java @@ -0,0 +1,121 @@ +package cucumber.runtime.junit; + +import cucumber.runtime.CucumberException; +import cucumber.runtime.Runtime; +import cucumber.runtime.model.CucumberFeature; +import cucumber.runtime.model.CucumberScenario; +import cucumber.runtime.model.CucumberScenarioOutline; +import cucumber.runtime.model.CucumberTagStatement; +import org.junit.runner.Description; +import org.junit.runner.notification.RunNotifier; +import org.junit.runners.ParentRunner; +import org.junit.runners.model.InitializationError; + +import java.util.ArrayList; +import java.util.List; + +public class SerenityFeatureRunner extends FeatureRunner { + private final List children = new ArrayList(); + + private int maxRetryCount = 0; + private int retries = 0; + private int scenarioCount = 0; + private Runtime runtime; + private CucumberFeature cucumberFeature; + private JUnitReporter jUnitReporter; + + public SerenityFeatureRunner( + CucumberFeature cucumberFeature, + Runtime runtime, + JUnitReporter jUnitReporter, + int maxRetryCount) + throws InitializationError { + super(cucumberFeature, runtime, jUnitReporter); + this.cucumberFeature = cucumberFeature; + this.runtime = runtime; + this.jUnitReporter = jUnitReporter; + this.maxRetryCount = maxRetryCount; + buildFeatureElementRunners(); + } + + private void buildFeatureElementRunners() { + for (CucumberTagStatement cucumberTagStatement : cucumberFeature.getFeatureElements()) { + try { + ParentRunner featureElementRunner; + if (cucumberTagStatement instanceof CucumberScenario) { + featureElementRunner = new ExecutionUnitRunner( + runtime, (CucumberScenario) cucumberTagStatement, jUnitReporter); + } else { + featureElementRunner = new SerenityScenarioOutlineRunner( + runtime, (CucumberScenarioOutline) cucumberTagStatement, jUnitReporter, maxRetryCount); + } + children.add(featureElementRunner); + } catch (InitializationError e) { + throw new CucumberException("Failed to create scenario runner", e); + } + } + } + + @Override + protected void runChild(ParentRunner child, RunNotifier notifier) { + child.run(notifier); + if(runtime.exitStatus() != 0) { + retry(notifier, child); + } + this.setScenarioCount(this.getScenarioCount() + 1); + retries = 0; + } + + private CucumberScenario getCurrentScenario() { + CucumberTagStatement cucumberTagStatement + = this.cucumberFeature.getFeatureElements().get(this.getScenarioCount()); + if (cucumberTagStatement instanceof CucumberScenarioOutline) { + return null; + } + return (CucumberScenario) cucumberTagStatement; + } + + public void retry(RunNotifier notifier, ParentRunner child) { + ParentRunner featureElementRunner = null; + CucumberScenario scenario = getCurrentScenario(); + if (scenario == null) { + return; + } + while (retries < maxRetryCount) { + try { + featureElementRunner = new ExecutionUnitRunner(runtime, scenario, jUnitReporter); + } catch (InitializationError e) { + throw new CucumberException("Failed to create scenario runner", e); + } + try { + featureElementRunner.run(notifier); + } catch(Throwable thr) { + thr.printStackTrace(); + } + if(runtime.exitStatus() == 0) { + break; + } else { + retries = retries + 1; + runtime.getErrors().clear(); + } + } + } + + @Override + protected List getChildren() { + return children; + } + + @Override + protected Description describeChild(ParentRunner child) { + return child.getDescription(); + } + + public int getScenarioCount() { + return scenarioCount; + } + + public void setScenarioCount(int scenarioCountValue) { + this.scenarioCount = scenarioCountValue; + } +} diff --git a/src/main/java/cucumber/runtime/junit/SerenityScenarioOutlineRunner.java b/src/main/java/cucumber/runtime/junit/SerenityScenarioOutlineRunner.java new file mode 100644 index 0000000..46f0389 --- /dev/null +++ b/src/main/java/cucumber/runtime/junit/SerenityScenarioOutlineRunner.java @@ -0,0 +1,34 @@ +package cucumber.runtime.junit; + +import cucumber.runtime.Runtime; +import cucumber.runtime.model.CucumberExamples; +import cucumber.runtime.model.CucumberScenarioOutline; +import org.junit.runner.Runner; +import org.junit.runners.Suite; +import org.junit.runners.model.InitializationError; + +import java.util.ArrayList; +import java.util.List; + +public class SerenityScenarioOutlineRunner extends Suite { + + public SerenityScenarioOutlineRunner( + Runtime runtimeValue, + CucumberScenarioOutline cucumberScenarioOutlineValue, + JUnitReporter jUnitReporterValue, + int retryCount) throws InitializationError { + super(null, buildRunners(runtimeValue, cucumberScenarioOutlineValue, jUnitReporterValue, retryCount)); + } + + private static List buildRunners( + Runtime runtime, + CucumberScenarioOutline cucumberScenarioOutline, + JUnitReporter jUnitReporter, + int retryCount) throws InitializationError { + List runners = new ArrayList(); + for (CucumberExamples cucumberExamples : cucumberScenarioOutline.getCucumberExamplesList()) { + runners.add(new SerenityExamplesRunner(runtime, cucumberExamples, jUnitReporter, retryCount)); + } + return runners; + } +} diff --git a/src/main/java/net/serenitybdd/cucumber/CucumberWithSerenity.java b/src/main/java/net/serenitybdd/cucumber/CucumberWithSerenity.java index 19ce0e5..45c55e5 100644 --- a/src/main/java/net/serenitybdd/cucumber/CucumberWithSerenity.java +++ b/src/main/java/net/serenitybdd/cucumber/CucumberWithSerenity.java @@ -3,37 +3,88 @@ import ch.lambdaj.Lambda; import ch.lambdaj.function.convert.Converter; import com.google.common.base.Splitter; -import cucumber.api.junit.Cucumber; import cucumber.runtime.ClassFinder; import cucumber.runtime.Runtime; import cucumber.runtime.RuntimeOptions; +import cucumber.runtime.RuntimeOptionsFactory; +import cucumber.runtime.io.MultiLoader; import cucumber.runtime.io.ResourceLoader; import cucumber.runtime.io.ResourceLoaderClassFinder; +import cucumber.runtime.junit.JUnitOptions; +import cucumber.runtime.junit.JUnitReporter; +import cucumber.runtime.junit.SerenityFeatureRunner; +import cucumber.runtime.model.CucumberFeature; import net.thucydides.core.ThucydidesSystemProperty; import net.thucydides.core.guice.Injectors; import net.thucydides.core.util.EnvironmentVariables; import net.thucydides.core.webdriver.Configuration; +import org.junit.runner.Description; +import org.junit.runner.notification.RunNotifier; +import org.junit.runners.ParentRunner; import org.junit.runners.model.InitializationError; +import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.List; +import static cucumber.runtime.junit.Assertions.assertNoCucumberAnnotatedMethods; +import static net.thucydides.core.ThucydidesSystemProperty.TEST_RETRY_COUNT; + /** * Glue code for running Cucumber via Thucydides. * Sets the Thucydides reporter. * * @author L.Carausu (liviu.carausu@gmail.com) */ -public class CucumberWithSerenity extends Cucumber { +public class CucumberWithSerenity extends ParentRunner { - public CucumberWithSerenity(Class clazz) throws InitializationError, IOException - { + private JUnitReporter jUnitReporter; + private List children = new ArrayList<>(); + private Runtime runtime; + private List cucumberFeatures; + private int maxRetryCount = 0; + + public CucumberWithSerenity(Class clazz) throws InitializationError, IOException { super(clazz); + initialize(clazz,Injectors.getInjector().getInstance(EnvironmentVariables.class)); + } + + public CucumberWithSerenity(Class clazz, EnvironmentVariables environmentVariables) throws InitializationError, IOException { + super(clazz); + initialize(clazz, environmentVariables); + } + + private void initialize(Class clazz,EnvironmentVariables environmentVariables) throws InitializationError, IOException { + maxRetryCount = TEST_RETRY_COUNT.integerFrom(environmentVariables, 0); + ClassLoader classLoader = clazz.getClassLoader(); + assertNoCucumberAnnotatedMethods(clazz); + RuntimeOptionsFactory runtimeOptionsFactory = new RuntimeOptionsFactory(clazz); + RuntimeOptions runtimeOptions = runtimeOptionsFactory.create(); + ResourceLoader resourceLoader = new MultiLoader(classLoader); + runtime = createRuntime(resourceLoader, classLoader, runtimeOptions); + final JUnitOptions junitOptions = new JUnitOptions(runtimeOptions.getJunitOptions()); + cucumberFeatures = runtimeOptions.cucumberFeatures(resourceLoader); + jUnitReporter = new JUnitReporter(runtimeOptions.reporter(classLoader), runtimeOptions.formatter(classLoader), runtimeOptions.isStrict(), junitOptions); + addChildren(cucumberFeatures,maxRetryCount); + } + + public CucumberWithSerenity withOutputDirectory(File outputDirectory) + { + Configuration systemConfiguration = Injectors.getInjector().getInstance(Configuration.class); + systemConfiguration.setOutputDirectory(outputDirectory); + return this; } /** - * Create the Runtime. Sets the Serenity runtime. + * Creates the Runtime. Sets the Serenity runtime. + * @param resourceLoader + * @param classLoader + * @param runtimeOptions + * @return cucumber Runtime + * @throws InitializationError + * @throws IOException */ protected cucumber.runtime.Runtime createRuntime(ResourceLoader resourceLoader, ClassLoader classLoader, @@ -42,7 +93,9 @@ protected cucumber.runtime.Runtime createRuntime(ResourceLoader resourceLoader, return createSerenityEnabledRuntime(resourceLoader, classLoader, runtimeOptions); } + private Collection environmentSpecifiedTags(List existingTags) { + EnvironmentVariables environmentVariables = Injectors.getInjector().getInstance(EnvironmentVariables.class); String tagsExpression = ThucydidesSystemProperty.TAGS.from(environmentVariables,""); List newTags = Lambda.convert(Splitter.on(",").trimResults().omitEmptyStrings().splitToList(tagsExpression), @@ -70,7 +123,7 @@ private Runtime createSerenityEnabledRuntime(ResourceLoader resourceLoader, ClassLoader classLoader, RuntimeOptions runtimeOptions) { Configuration systemConfiguration = Injectors.getInjector().getInstance(Configuration.class); - return createSerenityEnabledRuntime(resourceLoader, classLoader, runtimeOptions, systemConfiguration); + return createSerenityEnabledRuntime(resourceLoader, classLoader, runtimeOptions, systemConfiguration,maxRetryCount); } public static Runtime createSerenityEnabledRuntime(ResourceLoader resourceLoader, @@ -82,4 +135,43 @@ public static Runtime createSerenityEnabledRuntime(ResourceLoader resourceLoader runtimeOptions.addPlugin(reporter); return new Runtime(resourceLoader, classFinder, classLoader, runtimeOptions); } + + public static Runtime createSerenityEnabledRuntime(ResourceLoader resourceLoader, + ClassLoader classLoader, + RuntimeOptions runtimeOptions, + Configuration systemConfiguration, + int maxRetryCount) { + ClassFinder classFinder = new ResourceLoaderClassFinder(resourceLoader, classLoader); + SerenityReporter reporter = new SerenityReporter(systemConfiguration); + reporter.setMaxRetryCount(maxRetryCount); + runtimeOptions.addPlugin(reporter); + return new Runtime(resourceLoader, classFinder, classLoader, runtimeOptions); + } + + @Override + protected Description describeChild(SerenityFeatureRunner child) { + return child.getDescription(); + } + + @Override + protected void runChild(SerenityFeatureRunner child, RunNotifier notifier) { + child.run(notifier); + } + + private void addChildren(List cucumberFeatures,int retryCount) throws InitializationError { + children.clear(); + for (CucumberFeature cucumberFeature : cucumberFeatures) { + children.add(new SerenityFeatureRunner(cucumberFeature, runtime, jUnitReporter,retryCount)); + } + } + + @Override + public List getChildren() { + return children; + } + + public Runtime getRuntime() { + return runtime; + } + } diff --git a/src/main/java/net/serenitybdd/cucumber/SerenityReporter.java b/src/main/java/net/serenitybdd/cucumber/SerenityReporter.java index dab5d16..3f013aa 100644 --- a/src/main/java/net/serenitybdd/cucumber/SerenityReporter.java +++ b/src/main/java/net/serenitybdd/cucumber/SerenityReporter.java @@ -81,6 +81,8 @@ public class SerenityReporter implements Formatter, Reporter { private Optional forcedStoryResult = Optional.absent(); private Optional forcedScenarioResult = Optional.absent(); + private int retries = 0; + private int maxRetries = 0; private void clearStoryResult() { forcedStoryResult = Optional.absent(); @@ -90,6 +92,7 @@ private void clearScenarioResult() { forcedScenarioResult = Optional.absent(); } + private String currentTest; private boolean isPendingStory() { return ((forcedStoryResult.or(TestResult.UNDEFINED) == TestResult.PENDING) @@ -139,7 +142,6 @@ public void uri(String uri) { defaultFeatureName = Inflector.getInstance().humanize(defaultFeatureId); } - @Override public void feature(Feature feature) { @@ -283,7 +285,6 @@ private boolean getUniqueBrowserTagFrom(List tags) { @Override public void scenarioOutline(ScenarioOutline scenarioOutline) { addingScenarioOutlineSteps = true; -// startScenario(scenarioOutline.getName(), scenarioOutline.getDescription(), scenarioOutline.getTags()); } String currentScenarioId; @@ -398,15 +399,20 @@ public void startOfScenarioLifeCycle(Scenario scenario) { } startExample(); } else { - startScenario(scenario); + if(newScenario) { + startScenario(scenario); + } + else { //retry + retries++; + } } - } private void startScenario(Scenario scenario) { clearScenarioResult(); StepEventBus.getEventBus().setTestSource(StepEventBus.TEST_SOURCE_CUCUMBER); StepEventBus.getEventBus().testStarted(scenario.getName()); + currentTest = scenario.getName(); StepEventBus.getEventBus().addDescriptionToCurrentTest(scenario.getDescription()); StepEventBus.getEventBus().addTagsToCurrentTest(convertCucumberTags(currentFeature.getTags())); StepEventBus.getEventBus().addTagsToCurrentTest(convertCucumberTags(scenario.getTags())); @@ -495,11 +501,24 @@ public void endOfScenarioLifeCycle(Scenario scenario) { if (examplesRunning) { finishExample(); } else { - generateReports(); + if (lastTestFailed() && maxRetries > 0 && (retries < maxRetries)) { + StepEventBus.getEventBus().cancelPreviousTest(); + StepEventBus.getEventBus().testStarted(currentTest); + } else { + if(retries > 0) { + TestOutcome testOutcome = StepEventBus.getEventBus().getBaseStepListener().getTestOutcomes().get(getAllTestOutcomes().size() - 1); + if(testOutcome.isSuccess()) { + StepEventBus.getEventBus().lastTestPassedAfterRetries(retries + 1,new ArrayList()); + } + retries = 0; + } + generateReports(); + } } -// if (!useUniqueBrowser(scenario)) { -// ThucydidesWebDriverSupport.closeAllDrivers(); -// } + } + + private boolean lastTestFailed(){ + return getAllTestOutcomes().get(getAllTestOutcomes().size() - 1).getResult() == TestResult.FAILURE; } private boolean useUniqueBrowser(Scenario scenario) { @@ -611,7 +630,6 @@ public void result(Result result) { StepEventBus.getEventBus().testFinished(); } } - } private void updatePendingResults() { @@ -701,4 +719,8 @@ public List getAllTestOutcomes() { private String normalized(String value) { return value.replaceAll(OPEN_PARAM_CHAR, "{").replaceAll(CLOSE_PARAM_CHAR, "}"); } + + public void setMaxRetryCount(int maxRetryCount) { + this.maxRetries = maxRetryCount; + } } diff --git a/src/test/groovy/net/serenitybdd/cucumber/outcomes/WhenCreatingSerenityTestOutcomes.groovy b/src/test/groovy/net/serenitybdd/cucumber/outcomes/WhenCreatingSerenityTestOutcomes.groovy index dba5538..40933a2 100644 --- a/src/test/groovy/net/serenitybdd/cucumber/outcomes/WhenCreatingSerenityTestOutcomes.groovy +++ b/src/test/groovy/net/serenitybdd/cucumber/outcomes/WhenCreatingSerenityTestOutcomes.groovy @@ -1,7 +1,9 @@ package net.serenitybdd.cucumber.outcomes import com.github.goldin.spock.extensions.tempdir.TempDir +import net.serenitybdd.cucumber.CucumberWithSerenity import net.serenitybdd.cucumber.integration.* +import net.thucydides.core.ThucydidesSystemProperty import net.thucydides.core.model.TestOutcome import net.thucydides.core.model.TestResult import net.thucydides.core.model.TestStep @@ -9,15 +11,34 @@ import net.thucydides.core.model.TestTag import net.thucydides.core.reports.OutcomeFormat import net.thucydides.core.reports.TestOutcomeLoader import net.thucydides.core.steps.StepEventBus +import net.thucydides.core.util.MockEnvironmentVariables +import net.thucydides.core.webdriver.Configuration +import net.thucydides.core.webdriver.SystemPropertiesConfiguration +import org.junit.Before +import org.junit.runner.notification.RunNotifier +import org.mockito.MockitoAnnotations import spock.lang.Specification import static net.serenitybdd.cucumber.util.CucumberRunner.serenityRunnerForCucumberTestRunner +import static org.hamcrest.MatcherAssert.assertThat +import static org.hamcrest.Matchers.hasItem class WhenCreatingSerenityTestOutcomes extends Specification { @TempDir File outputDirectory + MockEnvironmentVariables environmentVariables; + + Configuration configuration; + + @Before + public void initMocks() { + MockitoAnnotations.initMocks(this); + environmentVariables = new MockEnvironmentVariables(); + configuration = new SystemPropertiesConfiguration(environmentVariables); + } + /* Feature: A simple feature @@ -73,12 +94,62 @@ class WhenCreatingSerenityTestOutcomes extends Specification { then: testOutcome.result == TestResult.FAILURE + stepResults.size() == 5 + recordedTestOutcomes.size() == 2 + and: + stepResults == [TestResult.SUCCESS,TestResult.SUCCESS,TestResult.SUCCESS,TestResult.FAILURE, TestResult.SKIPPED] + and: + testOutcome.testSteps[3].errorMessage.contains("expected:<[2]0> but was:<[1]0>") + } + + def "should record failures for a failing scenario with retries"() { + given: + environmentVariables.setProperty(ThucydidesSystemProperty.TEST_RETRY_COUNT.getPropertyName(), "3"); + def cucumber = new CucumberWithSerenity(FailingScenario.class,environmentVariables).withOutputDirectory(outputDirectory); + + when: + cucumber.run(new RunNotifier()) + List recordedTestOutcomes = new TestOutcomeLoader().forFormat(OutcomeFormat.JSON).loadFrom(outputDirectory).sort{it.name}; + TestOutcome testOutcome = recordedTestOutcomes[0] + List testSteps = testOutcome.testSteps; + List stepResults = testOutcome.testSteps.collect { step -> step.result } + + then: + testOutcome.result == TestResult.FAILURE + testSteps.size() == 5 + stepResults.size() == 5 + recordedTestOutcomes.size() == 2 and: stepResults == [TestResult.SUCCESS,TestResult.SUCCESS,TestResult.SUCCESS,TestResult.FAILURE, TestResult.SKIPPED] and: testOutcome.testSteps[3].errorMessage.contains("expected:<[2]0> but was:<[1]0>") } + def "should rerun the instable scenario and succeed"() { + given: + environmentVariables.setProperty(ThucydidesSystemProperty.TEST_RETRY_COUNT.getPropertyName(), "3"); + def cucumber = new CucumberWithSerenity(InstableScenario.class,environmentVariables).withOutputDirectory(outputDirectory); + + when: + cucumber.run(new RunNotifier()) + List recordedTestOutcomes = new TestOutcomeLoader().forFormat(OutcomeFormat.JSON).loadFrom(outputDirectory).sort{it.name}; + TestOutcome testOutcome = recordedTestOutcomes[0] + List testSteps = testOutcome.testSteps; + List stepResults = testOutcome.testSteps.collect { step -> step.result } + + then: + testOutcome.result == TestResult.SUCCESS + testSteps.size() == 5 + stepResults.size() == 5 + recordedTestOutcomes.size() == 1 + and: + stepResults == [TestResult.SUCCESS,TestResult.SUCCESS,TestResult.SUCCESS,TestResult.SUCCESS, TestResult.IGNORED] + and: + testSteps.get(4).getDescription().contains("UNSTABLE TEST") + and: + assertThat(testOutcome.getTags(), hasItem(TestTag.withName("Retries: 3").andType("unstable test"))); + } + def "should record a feature tag based on the name of the feature when the feature name is different from the feature file name"() { given: def runtime = serenityRunnerForCucumberTestRunner(SimpleScenarioWithALongName.class, outputDirectory); @@ -209,6 +280,7 @@ It goes for two lines""" def "should generate a well-structured Thucydides test outcome for feature files with several Cucumber scenario"() { given: + environmentVariables.setProperty(ThucydidesSystemProperty.TEST_RETRY_COUNT.getPropertyName(), "0"); def runtime = serenityRunnerForCucumberTestRunner(MultipleScenarios.class, outputDirectory); when: diff --git a/src/test/java/net/serenitybdd/cucumber/integration/InstableScenario.java b/src/test/java/net/serenitybdd/cucumber/integration/InstableScenario.java new file mode 100644 index 0000000..c56b9fc --- /dev/null +++ b/src/test/java/net/serenitybdd/cucumber/integration/InstableScenario.java @@ -0,0 +1,12 @@ +package net.serenitybdd.cucumber.integration; + +import cucumber.api.CucumberOptions; +import net.serenitybdd.cucumber.CucumberWithSerenity; +import org.junit.runner.RunWith; + +/** + * Created by liviu on 05/01/2017. + */ +@RunWith(CucumberWithSerenity.class) +@CucumberOptions(features="src/test/resources/samples/instable_scenario.feature") +public class InstableScenario {} diff --git a/src/test/java/net/serenitybdd/cucumber/integration/steps/SampleWidgetSteps.java b/src/test/java/net/serenitybdd/cucumber/integration/steps/SampleWidgetSteps.java index c85b679..f0cce7e 100644 --- a/src/test/java/net/serenitybdd/cucumber/integration/steps/SampleWidgetSteps.java +++ b/src/test/java/net/serenitybdd/cucumber/integration/steps/SampleWidgetSteps.java @@ -3,10 +3,11 @@ import cucumber.api.java.en.Given; import cucumber.api.java.en.Then; import cucumber.api.java.en.When; +import net.serenitybdd.cucumber.integration.steps.thucydides.WidgetRetrySteps; import net.serenitybdd.cucumber.integration.steps.thucydides.WidgetSteps; import net.thucydides.core.annotations.Steps; -import static org.assertj.core.api.Assertions.assertThat;; +; /** * Created by john on 23/07/2014. @@ -20,6 +21,9 @@ public class SampleWidgetSteps { @Steps WidgetSteps widgetSteps; + @Steps + WidgetRetrySteps widgetRetrySteps; + @Given("I have \\$(\\d+)") public void iHaveMoney(int money) { @@ -44,4 +48,9 @@ public void buyWidgets() { public void shouldBeBilled(int totalPrice) { widgetSteps.shouldBeBilled(billedPrice, totalPrice); } + + @Then("I should perhaps be billed \\$(\\d+)") + public void shouldPerhapsBeBilled(int totalPrice) { + widgetRetrySteps.aStepThatFailsOnMultipleOfFourTries(); + } } diff --git a/src/test/java/net/serenitybdd/cucumber/integration/steps/thucydides/WidgetRetrySteps.java b/src/test/java/net/serenitybdd/cucumber/integration/steps/thucydides/WidgetRetrySteps.java new file mode 100644 index 0000000..76a1522 --- /dev/null +++ b/src/test/java/net/serenitybdd/cucumber/integration/steps/thucydides/WidgetRetrySteps.java @@ -0,0 +1,17 @@ +package net.serenitybdd.cucumber.integration.steps.thucydides; + +import net.thucydides.core.annotations.Step; + +import static org.assertj.core.api.Assertions.assertThat; + + +public class WidgetRetrySteps { + + static int testCount = 1; + + @Step + public void aStepThatFailsOnMultipleOfFourTries() { + boolean shouldPass = testCount++ % 4 == 0; + assertThat(shouldPass).isTrue(); + } +} diff --git a/src/test/java/net/serenitybdd/cucumber/integration/steps/thucydides/WidgetSteps.java b/src/test/java/net/serenitybdd/cucumber/integration/steps/thucydides/WidgetSteps.java index 42d769f..4af2c86 100644 --- a/src/test/java/net/serenitybdd/cucumber/integration/steps/thucydides/WidgetSteps.java +++ b/src/test/java/net/serenitybdd/cucumber/integration/steps/thucydides/WidgetSteps.java @@ -1,7 +1,5 @@ package net.serenitybdd.cucumber.integration.steps.thucydides; -import cucumber.api.PendingException; -import cucumber.runtime.CucumberException; import net.thucydides.core.annotations.Step; import static org.assertj.core.api.Assertions.assertThat; diff --git a/src/test/resources/samples/instable_scenario.feature b/src/test/resources/samples/instable_scenario.feature new file mode 100644 index 0000000..23d7086 --- /dev/null +++ b/src/test/resources/samples/instable_scenario.feature @@ -0,0 +1,8 @@ +Feature: A simple feature that is instable + + @shouldFail + Scenario: A simple instable scenario + Given I want to purchase 2 widgets + And a widget costs $5 + When I buy the widgets + Then I should perhaps be billed $10 From d4b54b4108d458aa66ddd8089e73a2b5681babf4 Mon Sep 17 00:00:00 2001 From: cliviu Date: Fri, 6 Jan 2017 00:54:26 +0100 Subject: [PATCH 2/3] support for test.retry.count --- .../java/net/serenitybdd/cucumber/SerenityReporter.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/serenitybdd/cucumber/SerenityReporter.java b/src/main/java/net/serenitybdd/cucumber/SerenityReporter.java index 347c440..d69bddd 100644 --- a/src/main/java/net/serenitybdd/cucumber/SerenityReporter.java +++ b/src/main/java/net/serenitybdd/cucumber/SerenityReporter.java @@ -403,9 +403,11 @@ public void startOfScenarioLifeCycle(Scenario scenario) { } else { if(newScenario) { startScenario(scenario); - }SerenityReporter + } else { //retry - retries++; + if(maxRetries > 0) { + retries++; + } } } } From 2e105aa1adff2c842dab907e12a54b89c54c35e5 Mon Sep 17 00:00:00 2001 From: cliviu Date: Sun, 19 Feb 2017 23:03:20 +0100 Subject: [PATCH 3/3] support for test.retry.count.cucumber --- build.gradle | 2 +- .../net/serenitybdd/cucumber/CucumberWithSerenity.java | 6 +++--- .../java/net/serenitybdd/cucumber/SerenityReporter.java | 2 +- .../outcomes/WhenCreatingSerenityTestOutcomes.groovy | 8 ++++---- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/build.gradle b/build.gradle index d2bd55b..bc3ee70 100644 --- a/build.gradle +++ b/build.gradle @@ -44,7 +44,7 @@ ext { if (!project.hasProperty("bintrayApiKey")) { bintrayApiKey = '' } - serenityCoreVersion = '1.2.2' + serenityCoreVersion = '1.2.3-SNAPSHOT' cucumberJVMVersion = '1.2.5' versionCounter = new ProjectVersionCounter(isRelease: project.hasProperty("releaseBuild")) diff --git a/src/main/java/net/serenitybdd/cucumber/CucumberWithSerenity.java b/src/main/java/net/serenitybdd/cucumber/CucumberWithSerenity.java index 2b2808b..dcb9196 100644 --- a/src/main/java/net/serenitybdd/cucumber/CucumberWithSerenity.java +++ b/src/main/java/net/serenitybdd/cucumber/CucumberWithSerenity.java @@ -29,9 +29,9 @@ import java.util.Collection; import java.util.List; -import static cucumber.runtime.junit.Assertions.assertNoCucumberAnnotatedMethods; -import static net.thucydides.core.ThucydidesSystemProperty.TEST_RETRY_COUNT; import static ch.lambdaj.Lambda.on; +import static cucumber.runtime.junit.Assertions.assertNoCucumberAnnotatedMethods; +import static net.thucydides.core.ThucydidesSystemProperty.TEST_RETRY_COUNT_CUCUMBER; /** @@ -59,7 +59,7 @@ public CucumberWithSerenity(Class clazz, EnvironmentVariables environmentVariabl } private void initialize(Class clazz,EnvironmentVariables environmentVariables) throws InitializationError, IOException { - maxRetryCount = TEST_RETRY_COUNT.integerFrom(environmentVariables, 0); + maxRetryCount = TEST_RETRY_COUNT_CUCUMBER.integerFrom(environmentVariables, 0); ClassLoader classLoader = clazz.getClassLoader(); assertNoCucumberAnnotatedMethods(clazz); RuntimeOptionsFactory runtimeOptionsFactory = new RuntimeOptionsFactory(clazz); diff --git a/src/main/java/net/serenitybdd/cucumber/SerenityReporter.java b/src/main/java/net/serenitybdd/cucumber/SerenityReporter.java index d69bddd..06e1791 100644 --- a/src/main/java/net/serenitybdd/cucumber/SerenityReporter.java +++ b/src/main/java/net/serenitybdd/cucumber/SerenityReporter.java @@ -512,7 +512,7 @@ public void endOfScenarioLifeCycle(Scenario scenario) { if(retries > 0) { TestOutcome testOutcome = StepEventBus.getEventBus().getBaseStepListener().getTestOutcomes().get(getAllTestOutcomes().size() - 1); if(testOutcome.isSuccess()) { - StepEventBus.getEventBus().lastTestPassedAfterRetries(retries + 1,new ArrayList()); + StepEventBus.getEventBus().lastTestPassedAfterRetries(retries + 1,new ArrayList(),null); } retries = 0; } diff --git a/src/test/groovy/net/serenitybdd/cucumber/outcomes/WhenCreatingSerenityTestOutcomes.groovy b/src/test/groovy/net/serenitybdd/cucumber/outcomes/WhenCreatingSerenityTestOutcomes.groovy index 40933a2..08fb956 100644 --- a/src/test/groovy/net/serenitybdd/cucumber/outcomes/WhenCreatingSerenityTestOutcomes.groovy +++ b/src/test/groovy/net/serenitybdd/cucumber/outcomes/WhenCreatingSerenityTestOutcomes.groovy @@ -4,6 +4,7 @@ import com.github.goldin.spock.extensions.tempdir.TempDir import net.serenitybdd.cucumber.CucumberWithSerenity import net.serenitybdd.cucumber.integration.* import net.thucydides.core.ThucydidesSystemProperty +import net.thucydides.core.configuration.SystemPropertiesConfiguration import net.thucydides.core.model.TestOutcome import net.thucydides.core.model.TestResult import net.thucydides.core.model.TestStep @@ -13,7 +14,6 @@ import net.thucydides.core.reports.TestOutcomeLoader import net.thucydides.core.steps.StepEventBus import net.thucydides.core.util.MockEnvironmentVariables import net.thucydides.core.webdriver.Configuration -import net.thucydides.core.webdriver.SystemPropertiesConfiguration import org.junit.Before import org.junit.runner.notification.RunNotifier import org.mockito.MockitoAnnotations @@ -104,7 +104,7 @@ class WhenCreatingSerenityTestOutcomes extends Specification { def "should record failures for a failing scenario with retries"() { given: - environmentVariables.setProperty(ThucydidesSystemProperty.TEST_RETRY_COUNT.getPropertyName(), "3"); + environmentVariables.setProperty(ThucydidesSystemProperty.TEST_RETRY_COUNT_CUCUMBER.getPropertyName(), "3"); def cucumber = new CucumberWithSerenity(FailingScenario.class,environmentVariables).withOutputDirectory(outputDirectory); when: @@ -127,7 +127,7 @@ class WhenCreatingSerenityTestOutcomes extends Specification { def "should rerun the instable scenario and succeed"() { given: - environmentVariables.setProperty(ThucydidesSystemProperty.TEST_RETRY_COUNT.getPropertyName(), "3"); + environmentVariables.setProperty(ThucydidesSystemProperty.TEST_RETRY_COUNT_CUCUMBER.getPropertyName(), "3"); def cucumber = new CucumberWithSerenity(InstableScenario.class,environmentVariables).withOutputDirectory(outputDirectory); when: @@ -280,7 +280,7 @@ It goes for two lines""" def "should generate a well-structured Thucydides test outcome for feature files with several Cucumber scenario"() { given: - environmentVariables.setProperty(ThucydidesSystemProperty.TEST_RETRY_COUNT.getPropertyName(), "0"); + environmentVariables.setProperty(ThucydidesSystemProperty.TEST_RETRY_COUNT_CUCUMBER.getPropertyName(), "0"); def runtime = serenityRunnerForCucumberTestRunner(MultipleScenarios.class, outputDirectory); when: