diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 679a152..b42ee69 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,6 +23,8 @@ on: - README_TEMPLATE.md - CHANGELOG.md + workflow_dispatch: + env: VERSION_FILE: gradle.properties VERSION_EXTRACT_PATTERN: '(?<=version=).+' @@ -36,6 +38,9 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + with: + ref: 'master' + fetch-depth: 0 - name: Generate versions uses: HardNorth/github-version-generate@v1 @@ -51,7 +56,7 @@ jobs: java-version: '8' - name: Setup git credentials - uses: oleksiyrudenko/gha-git-credentials@v2.1.1 + uses: oleksiyrudenko/gha-git-credentials@v2-latest with: name: 'reportportal.io' email: 'support@reportportal.io' @@ -95,7 +100,6 @@ jobs: body: ${{ steps.readChangelogEntry.outputs.changes }} - name: Checkout develop branch - if: ${{github.ref}} == 'master' uses: actions/checkout@v4 with: ref: 'develop' @@ -103,7 +107,6 @@ jobs: - name: Merge release branch into develop id: mergeIntoDevelop - if: ${{github.ref}} == 'master' run: | git merge -m 'Merge master branch into develop after a release' origin/master git status | (! grep -Fq 'both modified:') || git status | grep -F 'both modified:' \ diff --git a/CHANGELOG.md b/CHANGELOG.md index a474cfe..7be6116 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ # Changelog ## [Unreleased] +### Added +- Common Stack Trace frames skip in description and logs, by @HardNorth +- Reporting of Last Error Log in Item description, by @HardNorth and @ArtemOAS +### Changed +- Client version updated on [5.2.22](https://github.com/reportportal/client-java/releases/tag/5.2.22), by @HardNorth ## [5.2.2] ### Changed diff --git a/build.gradle b/build.gradle index ae50276..aa43476 100644 --- a/build.gradle +++ b/build.gradle @@ -37,7 +37,7 @@ repositories { } dependencies { - api 'com.epam.reportportal:client-java:5.2.13' + api 'com.epam.reportportal:client-java:5.2.22' api "io.cucumber:cucumber-java:${project.cucumber_version}" implementation 'org.slf4j:slf4j-api:2.0.7' @@ -58,7 +58,7 @@ dependencies { testImplementation "org.junit.jupiter:junit-jupiter-api:${project.junit_version}" testImplementation "org.junit.jupiter:junit-jupiter-params:${project.junit_version}" testImplementation "org.junit.jupiter:junit-jupiter-engine:${project.junit_version}" - testImplementation 'org.apache.commons:commons-io:1.3.2' + testImplementation 'commons-io:commons-io:2.16.1' } test { diff --git a/src/main/java/com/epam/reportportal/cucumber/AbstractReporter.java b/src/main/java/com/epam/reportportal/cucumber/AbstractReporter.java index e5e0aa2..40078de 100644 --- a/src/main/java/com/epam/reportportal/cucumber/AbstractReporter.java +++ b/src/main/java/com/epam/reportportal/cucumber/AbstractReporter.java @@ -27,8 +27,8 @@ import com.epam.reportportal.service.tree.TestItemTree; import com.epam.reportportal.utils.*; import com.epam.reportportal.utils.files.ByteSource; +import com.epam.reportportal.utils.formatting.MarkdownUtils; import com.epam.reportportal.utils.http.ContentType; -import com.epam.reportportal.utils.markdown.MarkdownUtils; import com.epam.reportportal.utils.properties.SystemAttributesExtractor; import com.epam.reportportal.utils.reflect.Accessible; import com.epam.ta.reportportal.ws.model.FinishExecutionRQ; @@ -43,6 +43,7 @@ import io.cucumber.plugin.ConcurrentEventListener; import io.cucumber.plugin.event.*; import io.reactivex.Maybe; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -62,9 +63,10 @@ import static com.epam.reportportal.cucumber.Utils.*; import static com.epam.reportportal.cucumber.util.ItemTreeUtils.createKey; import static com.epam.reportportal.cucumber.util.ItemTreeUtils.retrieveLeaf; +import static com.epam.reportportal.utils.formatting.ExceptionUtils.getStackTrace; +import static java.lang.String.format; import static java.util.Optional.ofNullable; import static org.apache.commons.lang3.StringUtils.isNotBlank; -import static org.apache.commons.lang3.exception.ExceptionUtils.getStackTrace; /** * Abstract Cucumber 5.x formatter for Report Portal @@ -84,6 +86,7 @@ public abstract class AbstractReporter implements ConcurrentEventListener { private static final String METHOD_OPENING_BRACKET = "("; private static final String STEP_DEFINITION_FIELD_NAME = "stepDefinition"; private static final String DOCSTRING_DECORATOR = "\n\"\"\"\n"; + private static final String ERROR_FORMAT = "Error:\n%s"; public static final TestItemTree ITEM_TREE = new TestItemTree(); private static volatile ReportPortal REPORT_PORTAL = ReportPortal.builder().build(); @@ -100,6 +103,15 @@ public abstract class AbstractReporter implements ConcurrentEventListener { // End of feature occurs once launch is finished. private final Map featureEndTime = new ConcurrentHashMap<>(); + /** + * This map uses to record the description of the scenario and the step to append the error to the description. + */ + private final Map, String> descriptionsMap = new ConcurrentHashMap<>(); + /** + * This map uses to record errors to append to the description. + */ + private final Map, Throwable> errorMap = new ConcurrentHashMap<>(); + private final ThreadLocal currentScenarioContext = new ThreadLocal<>(); public static ReportPortal getReportPortal() { @@ -219,14 +231,20 @@ protected Maybe startScenario(@Nonnull Maybe featureId, @Nonnull * @param scenarioContext current scenario context */ protected void beforeScenario(RunningContext.FeatureContext featureContext, RunningContext.ScenarioContext scenarioContext) { - String scenarioName = Utils.buildName(scenarioContext.getKeyword(), + String scenarioName = Utils.buildName( + scenarioContext.getKeyword(), AbstractReporter.COLON_INFIX, scenarioContext.getTestCase().getName() ); - Maybe id = startScenario(featureContext.getFeatureId(), - buildStartScenarioRequest(scenarioContext.getTestCase(), scenarioName, featureContext.getUri(), scenarioContext.getLine()) + StartTestItemRQ startTestItemRQ = buildStartScenarioRequest( + scenarioContext.getTestCase(), + scenarioName, + featureContext.getUri(), + scenarioContext.getLine() ); + Maybe id = startScenario(featureContext.getFeatureId(), startTestItemRQ); scenarioContext.setId(id); + descriptionsMap.put(id, ofNullable(startTestItemRQ.getDescription()).orElse(StringUtils.EMPTY)); if (launch.get().getParameters().isCallbackReportingEnabled()) { addToTree(featureContext, scenarioContext); } @@ -247,6 +265,9 @@ protected void afterScenario(TestCaseFinished event) { RunningContext.ScenarioContext context = getCurrentScenarioContext(); URI featureUri = context.getFeatureUri(); currentScenarioContextMap.remove(Pair.of(context.getLine(), featureUri)); + if (mapItemStatus(event.getResult().getStatus()) == ItemStatus.FAILED) { + Optional.ofNullable(event.getResult().getError()).ifPresent(error -> errorMap.put(context.getId(), error)); + } Date endTime = finishTestItem(context.getId(), event.getResult().getStatus()); featureEndTime.put(featureUri, endTime); currentScenarioContext.remove(); @@ -353,7 +374,9 @@ protected void beforeStep(TestStep testStep) { context.setCurrentStepId(stepId); String stepText = step.getText(); context.setCurrentText(stepText); - + if (rq.isHasStats()) { + descriptionsMap.put(stepId, ofNullable(rq.getDescription()).orElse(StringUtils.EMPTY)); + } if (launch.get().getParameters().isCallbackReportingEnabled()) { addToTree(context, stepText, stepId); } @@ -367,6 +390,9 @@ protected void beforeStep(TestStep testStep) { protected void afterStep(Result result) { reportResult(result, null); RunningContext.ScenarioContext context = getCurrentScenarioContext(); + if (mapItemStatus(result.getStatus()) == ItemStatus.FAILED) { + Optional.ofNullable(result.getError()).ifPresent(error -> errorMap.put(context.getCurrentStepId(), error)); + } finishTestItem(context.getCurrentStepId(), result.getStatus()); context.setCurrentStepId(null); } @@ -466,7 +492,7 @@ protected void reportResult(@Nonnull Result result, @Nullable String message) { sendLog(message, level); } if (result.getError() != null) { - sendLog(getStackTrace(result.getError()), level); + sendLog(getStackTrace(result.getError(), new Throwable()), level); } } @@ -491,7 +517,8 @@ protected void embedding(@Nullable String name, @Nullable String mimeType, @Nonn String type = ofNullable(mimeType).filter(ContentType::isValidType).orElseGet(() -> getDataType(data, name)); String attachmentName = ofNullable(name).filter(m -> !m.isEmpty()) .orElseGet(() -> ofNullable(type).map(t -> t.substring(0, t.indexOf("/"))).orElse("")); - ReportPortal.emitLog(new ReportPortalMessage(ByteSource.wrap(data), type, attachmentName), + ReportPortal.emitLog( + new ReportPortalMessage(ByteSource.wrap(data), type, attachmentName), "UNKNOWN", Calendar.getInstance().getTime() ); @@ -616,14 +643,16 @@ protected void handleStartOfTestCase(@Nonnull TestCaseStarted event) { TestCase testCase = event.getTestCase(); RunningContext.FeatureContext newFeatureContext = new RunningContext.FeatureContext(testCase); URI featureUri = newFeatureContext.getUri(); - RunningContext.FeatureContext featureContext = currentFeatureContextMap.computeIfAbsent(featureUri, u -> { - getRootItemId(); // trigger root item creation - newFeatureContext.setFeatureId(startFeature(buildStartFeatureRequest(newFeatureContext.getFeature(), featureUri))); - if (launch.get().getParameters().isCallbackReportingEnabled()) { - addToTree(newFeatureContext); - } - return newFeatureContext; - }); + RunningContext.FeatureContext featureContext = currentFeatureContextMap.computeIfAbsent( + featureUri, u -> { + getRootItemId(); // trigger root item creation + newFeatureContext.setFeatureId(startFeature(buildStartFeatureRequest(newFeatureContext.getFeature(), featureUri))); + if (launch.get().getParameters().isCallbackReportingEnabled()) { + addToTree(newFeatureContext); + } + return newFeatureContext; + } + ); if (!featureContext.getUri().equals(testCase.getUri())) { throw new IllegalStateException("Scenario URI does not match Feature URI."); @@ -632,10 +661,12 @@ protected void handleStartOfTestCase(@Nonnull TestCaseStarted event) { RunningContext.ScenarioContext newScenarioContext = featureContext.getScenarioContext(testCase); Pair scenarioLineFeatureURI = Pair.of(newScenarioContext.getLine(), featureContext.getUri()); - RunningContext.ScenarioContext scenarioContext = currentScenarioContextMap.computeIfAbsent(scenarioLineFeatureURI, k -> { - currentScenarioContext.set(newScenarioContext); - return newScenarioContext; - }); + RunningContext.ScenarioContext scenarioContext = currentScenarioContextMap.computeIfAbsent( + scenarioLineFeatureURI, k -> { + currentScenarioContext.set(newScenarioContext); + return newScenarioContext; + } + ); beforeScenario(featureContext, scenarioContext); } @@ -664,14 +695,16 @@ protected void handleTestStepFinished(@Nonnull TestStepFinished event) { protected void addToTree(@Nonnull RunningContext.ScenarioContext scenarioContext, @Nullable String text, @Nonnull Maybe stepId) { - retrieveLeaf(scenarioContext.getFeatureUri(), + retrieveLeaf( + scenarioContext.getFeatureUri(), scenarioContext.getLine(), ITEM_TREE ).ifPresent(scenarioLeaf -> scenarioLeaf.getChildItems().put(createKey(text), TestItemTree.createTestItemLeaf(stepId))); } protected void removeFromTree(@Nonnull RunningContext.ScenarioContext scenarioContext, @Nullable String text) { - retrieveLeaf(scenarioContext.getFeatureUri(), + retrieveLeaf( + scenarioContext.getFeatureUri(), scenarioContext.getLine(), ITEM_TREE ).ifPresent(scenarioLeaf -> scenarioLeaf.getChildItems().remove(createKey(text))); @@ -686,15 +719,37 @@ protected void removeFromTree(@Nonnull RunningContext.ScenarioContext scenarioCo * @return finish request */ @Nonnull - @SuppressWarnings("unused") protected FinishTestItemRQ buildFinishTestItemRequest(@Nonnull Maybe itemId, @Nullable Date finishTime, @Nullable ItemStatus status) { FinishTestItemRQ rq = new FinishTestItemRQ(); + if (status == ItemStatus.FAILED) { + Optional currentDescription = Optional.ofNullable(descriptionsMap.remove(itemId)); + Optional currentError = Optional.ofNullable(errorMap.remove(itemId)); + currentDescription.flatMap(description -> currentError.map(errorMessage -> resolveDescriptionErrorMessage( + description, + errorMessage + ))).ifPresent(rq::setDescription); + } ofNullable(status).ifPresent(s -> rq.setStatus(s.name())); - rq.setEndTime(ofNullable(finishTime).orElse(Calendar.getInstance().getTime())); + rq.setEndTime(finishTime); return rq; } + /** + * Resolve description + * + * @param currentDescription Current description + * @param error Error message + * @return Description with error + */ + private String resolveDescriptionErrorMessage(String currentDescription, Throwable error) { + String errorStr = format(ERROR_FORMAT, getStackTrace(error, new Throwable())); + return Optional.ofNullable(currentDescription) + .filter(StringUtils::isNotBlank) + .map(description -> MarkdownUtils.asTwoParts(currentDescription, errorStr)) + .orElse(errorStr); + } + /** * Finish a feature with specific date and time * @@ -706,8 +761,10 @@ protected void finishFeature(@Nullable Maybe itemId, @Nullable Date date LOGGER.error("BUG: Trying to finish unspecified test item."); return; } + Date endTime = ofNullable(dateTime).orElse(Calendar.getInstance().getTime()); + FinishTestItemRQ rq = buildFinishTestItemRequest(itemId, endTime, null); //noinspection ReactiveStreamsUnusedPublisher - launch.get().finishTestItem(itemId, buildFinishTestItemRequest(itemId, dateTime, null)); + launch.get().finishTestItem(itemId, rq); } /** @@ -723,11 +780,11 @@ protected Date finishTestItem(@Nullable Maybe itemId, @Nullable Status s LOGGER.error("BUG: Trying to finish unspecified test item."); return null; } - FinishTestItemRQ rq = buildFinishTestItemRequest(itemId, null, mapItemStatus(status)); - new FinishTestItemRQ(); + Date endTime = Calendar.getInstance().getTime(); + FinishTestItemRQ rq = buildFinishTestItemRequest(itemId, endTime, mapItemStatus(status)); //noinspection ReactiveStreamsUnusedPublisher launch.get().finishTestItem(itemId, rq); - return rq.getEndTime(); + return endTime; } /** @@ -751,7 +808,8 @@ protected ItemStatus mapItemStatus(@Nullable Status status) { return null; } else { if (STATUS_MAPPING.get(status) == null) { - LOGGER.error(String.format("Unable to find direct mapping between Cucumber and ReportPortal for TestItem with status: '%s'.", + LOGGER.error(String.format( + "Unable to find direct mapping between Cucumber and ReportPortal for TestItem with status: '%s'.", status )); return ItemStatus.SKIPPED; @@ -954,7 +1012,8 @@ protected TestCaseIdEntry getTestCaseId(@Nonnull TestStep testStep, @Nullable St if (definitionMatch != null) { try { Method method = retrieveMethod(definitionMatch); - return TestCaseIdUtils.getTestCaseId(method.getAnnotation(TestCaseId.class), + return TestCaseIdUtils.getTestCaseId( + method.getAnnotation(TestCaseId.class), method, codeRef, (List) ARGUMENTS_TRANSFORM.apply(arguments) diff --git a/src/test/java/com/epam/reportportal/cucumber/FailedTest.java b/src/test/java/com/epam/reportportal/cucumber/FailedTest.java index b7292c9..8ed2dda 100644 --- a/src/test/java/com/epam/reportportal/cucumber/FailedTest.java +++ b/src/test/java/com/epam/reportportal/cucumber/FailedTest.java @@ -25,6 +25,7 @@ import com.epam.reportportal.service.ReportPortal; import com.epam.reportportal.service.ReportPortalClient; import com.epam.reportportal.util.test.CommonUtils; +import com.epam.reportportal.utils.formatting.MarkdownUtils; import com.epam.ta.reportportal.ws.model.FinishTestItemRQ; import com.epam.ta.reportportal.ws.model.log.SaveLogRQ; import io.cucumber.testng.AbstractTestNGCucumberTests; @@ -42,8 +43,8 @@ import static com.epam.reportportal.cucumber.integration.util.TestUtils.filterLogs; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.*; +import static org.mockito.Mockito.any; import static org.mockito.Mockito.*; /** @@ -52,18 +53,28 @@ public class FailedTest { private static final String EXPECTED_ERROR = "java.lang.IllegalStateException: " + FailedSteps.ERROR_MESSAGE; + private static final String EXPECTED_STACK_TRACE = EXPECTED_ERROR + + "\n\tat com.epam.reportportal.cucumber.integration.feature.FailedSteps.i_have_a_failed_step(FailedSteps.java:31)" + + "\n\tat ✽.I have a failed step(file://" + System.getProperty("user.dir") + + "/src/test/resources/features/FailedScenario.feature:4)\n"; + private static final String ERROR_LOG_TEXT = "Error:\n" + EXPECTED_STACK_TRACE; + + private static final String SCENARIO_CODE_REFERENCES_WITH_ERROR = MarkdownUtils.asTwoParts( + "file://" + System.getProperty("user.dir") + "/src/test/resources/features/FailedScenario.feature", + ERROR_LOG_TEXT + ); @CucumberOptions(features = "src/test/resources/features/FailedScenario.feature", glue = { "com.epam.reportportal.cucumber.integration.feature" }, plugin = { "pretty", "com.epam.reportportal.cucumber.integration.TestScenarioReporter" }) - public static class FailedScenarioReporter extends AbstractTestNGCucumberTests { + public static class FailedScenarioReporterTest extends AbstractTestNGCucumberTests { } @CucumberOptions(features = "src/test/resources/features/FailedScenario.feature", glue = { "com.epam.reportportal.cucumber.integration.feature" }, plugin = { "pretty", "com.epam.reportportal.cucumber.integration.TestStepReporter" }) - public static class FailedStepReporter extends AbstractTestNGCucumberTests { + public static class FailedStepReporterTest extends AbstractTestNGCucumberTests { } @@ -91,7 +102,7 @@ public void initLaunch() { @Test @SuppressWarnings("unchecked") public void verify_failed_step_reporting_scenario_reporter() { - TestUtils.runTests(FailedScenarioReporter.class); + TestUtils.runTests(FailedScenarioReporterTest.class); verify(client).startTestItem(any()); verify(client).startTestItem(same(suiteId), any()); @@ -117,7 +128,7 @@ public void verify_failed_step_reporting_scenario_reporter() { @Test @SuppressWarnings("unchecked") public void verify_failed_step_reporting_step_reporter() { - TestUtils.runTests(FailedStepReporter.class); + TestUtils.runTests(FailedStepReporterTest.class); verify(client).startTestItem(any()); verify(client).startTestItem(same(suiteId), any()); @@ -136,4 +147,44 @@ public void verify_failed_step_reporting_step_reporter() { SaveLogRQ expectedError = expectedErrorList.get(0); assertThat(expectedError.getItemUuid(), equalTo(stepId)); } + + @Test + public void verify_failed_nested_step_description_scenario_reporter() { + TestUtils.runTests(FailedScenarioReporterTest.class); + + verify(client).startTestItem(any()); + verify(client).startTestItem(same(suiteId), any()); + verify(client).startTestItem(same(testId), any()); + verify(client).startTestItem(same(stepId), any()); + ArgumentCaptor finishCaptor = ArgumentCaptor.forClass(FinishTestItemRQ.class); + verify(client).finishTestItem(same(nestedStepId), finishCaptor.capture()); + verify(client).finishTestItem(same(stepId), any()); + verify(client).finishTestItem(same(testId), finishCaptor.capture()); + + List finishRqs = finishCaptor.getAllValues(); + finishRqs.subList(0, finishRqs.size() - 1).forEach(e -> assertThat(e.getStatus(), equalTo(ItemStatus.FAILED.name()))); + + FinishTestItemRQ step = finishRqs.get(0); + assertThat(step.getDescription(), not(equalTo(ERROR_LOG_TEXT))); + } + + @Test + public void verify_failed_step_description_step_reporter() { + TestUtils.runTests(FailedStepReporterTest.class); + + verify(client).startTestItem(any()); + verify(client).startTestItem(same(suiteId), any()); + verify(client).startTestItem(same(testId), any()); + ArgumentCaptor finishCaptor = ArgumentCaptor.forClass(FinishTestItemRQ.class); + verify(client).finishTestItem(same(stepId), finishCaptor.capture()); + verify(client).finishTestItem(same(testId), finishCaptor.capture()); + + List finishRqs = finishCaptor.getAllValues(); + finishRqs.forEach(e -> assertThat(e.getStatus(), equalTo(ItemStatus.FAILED.name()))); + + FinishTestItemRQ step = finishRqs.get(0); + assertThat(step.getDescription(), equalTo(ERROR_LOG_TEXT)); + FinishTestItemRQ test = finishRqs.get(1); + assertThat(test.getDescription(), equalTo((SCENARIO_CODE_REFERENCES_WITH_ERROR))); + } }