From 19a1b6f9425b3a63b396bb8d947b4c403e020e20 Mon Sep 17 00:00:00 2001 From: minwoox Date: Wed, 27 Nov 2024 22:39:41 +0900 Subject: [PATCH] Add reportFailedTest that we use for Armeria I realized that we don't have reportFailedTest which is very convenient for tracking especially flaky tests. This PR copies all the work in https://github.com/line/armeria/pull/4292 --- .github/workflows/actions_build.yml | 30 ++++++++-- build.gradle | 91 +++++++++++++++++++++++++++++ dependencies.toml | 6 ++ 3 files changed, 121 insertions(+), 6 deletions(-) diff --git a/.github/workflows/actions_build.yml b/.github/workflows/actions_build.yml index 26f77ec737..2dd109fd4f 100644 --- a/.github/workflows/actions_build.yml +++ b/.github/workflows/actions_build.yml @@ -75,11 +75,34 @@ jobs: -Porg.gradle.java.installations.paths=${{ steps.setup-build-jdk.outputs.path }},${{ steps.setup-test-jdk.outputs.path }} shell: bash + - name: Summarize the failed tests + if: failure() + run: | + ./gradlew --no-daemon --stacktrace --max-workers=1 reportFailedTests \ + -PnoLint \ + -PflakyTests=false \ + -PbuildJdkVersion=${{ env.BUILD_JDK_VERSION }} \ + -PtestJavaVersion=${{ matrix.java }} \ + ${{ matrix.min-java && format('-PminimumJavaVersion={0}', matrix.min-java) || '' }} \ + -Porg.gradle.java.installations.paths=${{ steps.setup-build-jdk.outputs.path }},${{ steps.setup-jdk.outputs.path }} + + SUMMARY_FILE="build/failed-tests-result.txt" + if test -f "$SUMMARY_FILE"; then + echo '### 🔴 Failed tests' >> $GITHUB_STEP_SUMMARY + cat $SUMMARY_FILE >> $GITHUB_STEP_SUMMARY + fi + shell: bash + + - name: Dump stuck threads + if: always() + run: jps | grep -vi "jps" | awk '{ print $1 }' | xargs -I'{}' jstack -l {} || true + shell: bash + - name: Upload coverage to Codecov if: ${{ matrix.coverage }} uses: codecov/codecov-action@v3 - - name: Collecting the test reports .. + - name: Collect the test reports .. if: failure() run: | find . '(' \ @@ -96,11 +119,6 @@ jobs: path: reports-JVM-${{ matrix.java }}.tar retention-days: 3 - - name: Dump stuck threads - if: always() - run: jps | grep -vi "jps" | awk '{ print $1 }' | xargs -I'{}' jstack -l {} || true - shell: bash - lint: if: github.repository == 'line/centraldogma' runs-on: ubuntu-latest diff --git a/build.gradle b/build.gradle index 06c3b008b0..e8e111470a 100644 --- a/build.gradle +++ b/build.gradle @@ -1,3 +1,15 @@ +import org.jsoup.Jsoup +import org.jsoup.nodes.Element +import org.jsoup.select.Elements + +import static java.lang.Math.min + +buildscript { + dependencies { + classpath libs.jsoup + } +} + plugins { alias libs.plugins.nexus.publish alias libs.plugins.osdetector apply false @@ -127,6 +139,85 @@ configure(projectsWithFlags('java')) { tasks.processResources.duplicatesStrategy = DuplicatesStrategy.INCLUDE } +tasks.register("reportFailedTests", TestsReportTask) + +/** + * Summarizes the failed tests and reports as a file with the Markdown syntax. + */ +class TestsReportTask extends DefaultTask { + @OutputFile + final def reportFile = project.file("${project.buildDir}/failed-tests-result.txt") + + @TaskAction + def run() { + // Collect up to 20 error results + int maxErrorSize = 20 + List failedTests = [] + Set handledFiles = [] + + project.allprojects { + tasks.withType(Test) { testTask -> + + def xmlFiles = testTask.reports.junitXml.outputLocation.asFileTree.files + if (xmlFiles.isEmpty()) { + return + } + xmlFiles.each { file -> + if (!handledFiles.add(file.name)) { + return + } + + Elements failures = Jsoup.parse(file, 'UTF-8').select("testsuite failure") + if (failures.isEmpty() || failedTests.size() > maxErrorSize) { + return + } + failures.each { failure -> + Element parent = failure.parent() + String fullMethodName = "${parent.attr("classname")}.${parent.attr("name")}" + String detail = failure.wholeText() + failedTests += [method: fullMethodName, detail: detail] + } + } + } + } + + if (failedTests.isEmpty()) { + return + } + + reportFile.withPrintWriter('UTF-8') { writer -> + failedTests.each { it -> + String method = it.method + String detail = it.detail + + // Create an link to directly create an issue from the error message + String ghIssueTitle = URLEncoder.encode("Test failure: `$method`", "UTF-8") + // 8k is the maximum allowed URL length for GitHub + String ghIssueBody = URLEncoder.encode( + "```\n${detail.substring(0, min(6000, detail.length()))}\n```\n", "UTF-8") + String ghIssueLink = + "https://github.com/line/centraldogma/issues/new?title=$ghIssueTitle&body=$ghIssueBody" + String ghSearchQuery = URLEncoder.encode("is:issue $method", "UTF-8") + String ghSearchLink = "https://github.com/line/centraldogma/issues?q=$ghSearchQuery" + writer.print("- $it.method - [Search similar issues]($ghSearchLink) | ") + writer.println("[Create an issue?]($ghIssueLink)") + + writer.println(" ```") + List lines = detail.split("\n") as List + def summary = lines.take(8) + summary.each { line -> writer.println(" $line") } + writer.println(" ```") + if (lines.size() > 8) { + writer.println("
Full error messages") + writer.println(" 
")
+                    lines.each { line -> writer.println("  $line") }
+                    writer.println("
\n") + } + } + } + } +} + // Configure the Javadoc tasks of all projects. allprojects { tasks.withType(Javadoc) { diff --git a/dependencies.toml b/dependencies.toml index ed70c06ef2..6486d5ced1 100644 --- a/dependencies.toml +++ b/dependencies.toml @@ -49,6 +49,7 @@ jsch = "0.1.55" json-path = "2.2.0" # 3.0.0 requires java 17 json-unit = "2.38.0" +jsoup = "1.18.1" jmh-core = "1.37" jmh-gradle-plugin = "0.7.2" jxr = "0.2.1" @@ -281,6 +282,11 @@ version.ref = "json-unit" module = "net.javacrumbs.json-unit:json-unit-fluent" version.ref = "json-unit" +# JSoup is only used for Gradle script and SAML testing. +[libraries.jsoup] +module = "org.jsoup:jsoup" +version.ref = "jsoup" + [libraries.junit4] module = "junit:junit" version.ref = "junit4"