feat: ignore flaky tests

README.md

@@ -33,6 +33,10 @@ Optional. Check will fail if there are test failures. The default is `false`.
 
 Optional. Check will fail if no tests were found. The default is `true`.
 
+### `ignore_flaky_tests`
+
+Optional. Set to `true` to consider flaky tests as success. The default is `false`.
+
 ### `skip_publishing`
 
 Optional. Skip the test report publishing (check run creation). The default is `false`.

action.js

@@ -15,11 +15,12 @@ const action = async () => {
     const commit = core.getInput('commit');
     const failOnFailedTests = core.getInput('fail_on_test_failures') === 'true';
     const failIfNoTests = core.getInput('fail_if_no_tests') === 'true';
+    const ignoreFlakyTests = core.getInput('ignore_flaky_tests') === 'true';
     const skipPublishing = core.getInput('skip_publishing') === 'true';
     const isFilenameInStackTrace = core.getInput('file_name_in_stack_trace') === 'true';
     const githubBaseUrl = core.getInput('github_base_url');
 
-    let { count, skipped, annotations } = await parseTestReports(reportPaths, isFilenameInStackTrace);
+    let { count, skipped, annotations } = await parseTestReports(reportPaths, isFilenameInStackTrace, ignoreFlakyTests);
     const foundResults = count > 0 || skipped > 0;
     const conclusion =
         (foundResults && annotations.length === 0) || (!foundResults && !failIfNoTests)

action.test.js

@@ -107,6 +107,24 @@ describe('action should work', () => {
         expect(failed).toBeNull();
     });
 
+    it('should send all ok if tests were flaky and ignore_flaky_test is true', async () => {
+        inputs.report_paths = '**/surefire-reports/TEST-*AllOkWithFlakesTest.xml';
+        inputs.ignore_flaky_tests = 'true';
+        let request = null;
+        const scope = nock('https://api.github.com')
+            .post('/repos/scacap/action-surefire-report/check-runs', body => {
+                request = body;
+                return body;
+            })
+            .reply(200, {});
+        await action();
+        scope.done();
+
+        expect(request).toStrictEqual(finishedSuccess);
+        expect(outputs).toHaveProperty('conclusion', 'success');
+        expect(failed).toBeNull();
+    });
+
     it('should send failure if no test results were found', async () => {
         inputs.report_paths = '**/xxx/*.xml';
         let request = null;

action.yml

@@ -31,6 +31,10 @@ inputs:
     description: 'fail run if there were no test results found'
     required: false
     default: 'true'
+  ignore_flaky_tests:
+    description: 'consider flaky tests as success'
+    required: false
+    default: 'false'
   skip_publishing:
     description: 'skip test report publishing'
     required: false

integration-tests/maven/flakes/pom.xml

@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>tests</artifactId>
+        <groupId>action.surefire.report</groupId>
+        <version>0.0.1-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+    <artifactId>flakes</artifactId>
+
+    <dependencies>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.hamcrest</groupId>
+            <artifactId>hamcrest-all</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <configuration>
+                    <argLine>-Duser.language=en</argLine>
+                    <redirectTestOutputToFile>true</redirectTestOutputToFile>
+                    <testFailureIgnore>true</testFailureIgnore>
+                    <rerunFailingTestsCount>1</rerunFailingTestsCount>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>compile</id>
+                        <phase>compile</phase>
+                        <goals>
+                            <goal>compile</goal>
+                        </goals>
+                    </execution>
+                    <execution>
+                        <id>testCompile</id>
+                        <phase>test-compile</phase>
+                        <goals>
+                            <goal>testCompile</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+
+</project>

integration-tests/maven/flakes/src/test/java/action/surefire/report/calc/AllOkWithFlakesTest.java

@@ -0,0 +1,20 @@
+package action.surefire.report.calc;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+
+public class AllOkWithFlakesTest {
+
+    private static boolean failTest = true;
+
+    @Test
+    public void firstTryFailSecondTrySuccess() {
+        if(failTest) {
+            failTest = false;
+            assertTrue(false);
+        } else {
+            assertTrue(true);
+        }
+    }
+}

integration-tests/maven/pom.xml

@@ -13,6 +13,7 @@
         <module>utils</module>
         <module>email</module>
         <module>evil_twins</module>
+        <module>flakes</module>
     </modules>
 
     <properties>

utils.js

@@ -62,7 +62,7 @@ function getTestsuites(report) {
     return [report.testsuites.testsuite];
 }
 
-async function parseFile(file, isFilenameInStackTrace) {
+async function parseFile(file, isFilenameInStackTrace, ignoreFlakyTests) {
     core.debug(`Parsing file ${file}`);
     let count = 0;
     let skipped = 0;
@@ -85,7 +85,7 @@ async function parseFile(file, isFilenameInStackTrace) {
         for (const testcase of testcases) {
             count++;
             if (testcase.skipped) skipped++;
-            if (testcase.failure || testcase.flakyFailure || testcase.error) {
+            if (testcase.failure || (testcase.flakyFailure && !ignoreFlakyTests) || testcase.error) {
                 let testcaseData =
                     (testcase.failure && testcase.failure._cdata) ||
                     (testcase.failure && testcase.failure._text) ||
@@ -139,13 +139,13 @@ async function parseFile(file, isFilenameInStackTrace) {
     return {count, skipped, annotations};
 }
 
-const parseTestReports = async (reportPaths, isFilenameInStackTrace) => {
+const parseTestReports = async (reportPaths, isFilenameInStackTrace, ignoreFlakyTests) => {
     const globber = await glob.create(reportPaths, {followSymbolicLinks: false});
     let annotations = [];
     let count = 0;
     let skipped = 0;
     for await (const file of globber.globGenerator()) {
-        const {count: c, skipped: s, annotations: a} = await parseFile(file, isFilenameInStackTrace);
+        const {count: c, skipped: s, annotations: a} = await parseFile(file, isFilenameInStackTrace, ignoreFlakyTests);
         if (c === 0) continue;
         count += c;
         skipped += s;