Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add retried to output #1238

Merged
merged 2 commits into from
Nov 28, 2024
Merged

Add retried to output #1238

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
eb88944
set-output name=retried
saturninoabril Nov 18, 2024
319aaf3
- rebase and update testcase with new inputs
mikepenz Nov 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 66 additions & 0 deletions __tests__/testParser.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1269,6 +1269,71 @@ describe('parseTestReports', () => {
])
})

it('should parse retried tests', async () => {
const {checkName, summary, totalCount, skipped, failed, passed, retried, globalAnnotations} =
await parseTestReports(
'checkName',
'summary',
'test_results/junit-server-test/report.xml',
'*',
true,
true,
true,
[],
'{{SUITE_NAME}}/{{TEST_NAME}}',
'/'
)

expect(checkName).toBe('checkName')
expect(summary).toBe('summary')
expect(totalCount).toBe(3)
expect(skipped).toBe(1)
expect(failed).toBe(0)
expect(passed).toBe(2)
expect(retried).toBe(1)
expect(globalAnnotations).toStrictEqual([
{
path: 'com/example/example/server/v8/channels/api4',
start_line: 1,
end_line: 1,
start_column: 0,
end_column: 0,
retries: 1,
annotation_level: 'notice',
status: 'success',
title: 'github.com/example/example/server/v8/channels/api4/TestWebSocketReconnectRace',
message: 'TestWebSocketReconnectRace',
raw_details: ''
},
{
annotation_level: 'notice',
end_column: 0,
end_line: 1,
message: 'TestCreateChannelBookmark',
path: 'com/example/example/server/v8/channels/api4',
raw_details: '',
retries: 0,
start_column: 0,
start_line: 1,
status: 'skipped',
title: 'github.com/example/example/server/v8/channels/api4/TestCreateChannelBookmark'
},
{
annotation_level: 'notice',
end_column: 0,
end_line: 1,
message: 'TestWebSocketUpgrade',
path: 'com/example/example/server/v8/channels/api4',
raw_details: '',
retries: 0,
start_column: 0,
start_line: 1,
status: 'success',
title: 'github.com/example/example/server/v8/channels/api4/TestWebSocketUpgrade'
}
])
})

it('parse corrupt test output', async () => {
const result = await parseTestReports(
'',
Expand Down Expand Up @@ -1296,6 +1361,7 @@ describe('parseTestReports', () => {
foundFiles: 1,
globalAnnotations: [],
passed: 0,
retried: 0,
testResults: []
})
})
Expand Down
20 changes: 18 additions & 2 deletions dist/index.js
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion dist/index.js.map
View file
Open in desktop

Large diffs are not rendered by default.

3 changes: 3 additions & 0 deletions src/main.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ export async function run(): Promise<void> {
skipped: 0,
failed: 0,
passed: 0,
retried: 0,
foundFiles: 0,
globalAnnotations: [],
testResults: []
Expand Down Expand Up @@ -96,13 +97,15 @@ export async function run(): Promise<void> {
mergedResult.skipped += testResult.skipped
mergedResult.failed += testResult.failed
mergedResult.passed += testResult.passed
mergedResult.retried += testResult.retried
testResults.push(testResult)
}

core.setOutput('total', mergedResult.totalCount)
core.setOutput('passed', mergedResult.passed)
core.setOutput('skipped', mergedResult.skipped)
core.setOutput('failed', mergedResult.failed)
core.setOutput('retried', mergedResult.retried)

if (!(mergedResult.totalCount > 0 || mergedResult.skipped > 0) && requireTests) {
core.setFailed(`❌ No test results found for ${checkName}`)
Expand Down
20 changes: 18 additions & 2 deletions src/testParser.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ export interface ActualTestResult {
name: string
totalCount: number
skippedCount: number
retriedCount: number
annotations: Annotation[]
globalAnnotations: Annotation[]
testResults: ActualTestResult[]
Expand All @@ -17,6 +18,7 @@ export interface ActualTestResult {
interface TestCasesResult {
totalCount: number
skippedCount: number
retriedCount: number
annotations: Annotation[]
}

Expand All @@ -27,6 +29,7 @@ export interface TestResult {
skipped: number
failed: number
passed: number
retried: number
foundFiles: number
globalAnnotations: Annotation[]
testResults: ActualTestResult[]
Expand Down Expand Up @@ -271,6 +274,7 @@ async function parseSuite(

let totalCount = 0
let skippedCount = 0
let retriedCount = 0
const annotations: Annotation[] = []

// parse testCases
Expand Down Expand Up @@ -300,6 +304,7 @@ async function parseSuite(
// expand global annotations array
totalCount += parsedTestCases.totalCount
skippedCount += parsedTestCases.skippedCount
retriedCount += parsedTestCases.retriedCount
annotations.push(...parsedTestCases.annotations)
globalAnnotations.push(...parsedTestCases.annotations)
}
Expand All @@ -309,6 +314,7 @@ async function parseSuite(
name: suiteName,
totalCount,
skippedCount,
retriedCount,
annotations,
globalAnnotations,
testResults: []
Expand Down Expand Up @@ -349,6 +355,7 @@ async function parseSuite(
childSuiteResults.push(childSuiteResult)
totalCount += childSuiteResult.totalCount
skippedCount += childSuiteResult.skippedCount
retriedCount += childSuiteResult.retriedCount
}

// skip out if we reached our annotations limit
Expand All @@ -357,6 +364,7 @@ async function parseSuite(
name: suiteName,
totalCount,
skippedCount,
retriedCount,
annotations,
globalAnnotations,
testResults: childSuiteResults
Expand All @@ -368,6 +376,7 @@ async function parseSuite(
name: suiteName,
totalCount,
skippedCount,
retriedCount,
annotations,
globalAnnotations,
testResults: childSuiteResults
Expand All @@ -394,6 +403,7 @@ async function parseTestCases(
const annotations: Annotation[] = []
let totalCount = 0
let skippedCount = 0
let retriedCount = 0
if (checkRetries) {
// identify duplicates, in case of flaky tests, and remove them
const testcaseMap = new Map<string, any>()
Expand All @@ -407,12 +417,14 @@ async function parseTestCases(
if (failed && !previousFailed) {
// previous is a success, drop failure
previous.retries = (previous.retries || 0) + 1
retriedCount += 1
core.debug(`Drop flaky test failure for (1): ${key}`)
} else if (!failed && previousFailed) {
// previous failed, new one not, replace
testcase.retries = (previous.retries || 0) + 1
testcaseMap.set(key, testcase)
core.debug(`Drop flaky test failure for (2): ${key}`)
retriedCount += 1
core.debug(`Drop flaky test failure for (2): ${JSON.stringify(testcase)}`)
}
} else {
testcaseMap.set(key, testcase)
Expand Down Expand Up @@ -551,6 +563,7 @@ async function parseTestCases(
return {
totalCount,
skippedCount,
retriedCount,
annotations
}
}
Expand Down Expand Up @@ -586,6 +599,7 @@ export async function parseTestReports(
const testResults: ActualTestResult[] = []
let totalCount = 0
let skipped = 0
let retried = 0
let foundFiles = 0
for await (const file of globber.globGenerator()) {
foundFiles++
Expand All @@ -610,9 +624,10 @@ export async function parseTestReports(
)

if (!testResult) continue
const {totalCount: c, skippedCount: s} = testResult
const {totalCount: c, skippedCount: s, retriedCount: r} = testResult
totalCount += c
skipped += s
retried += r
testResults.push(testResult)

if (annotationsLimit > 0 && globalAnnotations.length >= annotationsLimit) {
Expand All @@ -631,6 +646,7 @@ export async function parseTestReports(
skipped,
failed,
passed,
retried,
foundFiles,
globalAnnotations,
testResults
Expand Down
16 changes: 16 additions & 0 deletions test_results/junit-server-test/report.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="3" failures="1" errors="0" time="24.733407">
<testsuite tests="3" failures="1" time="10.715000" name="github.com/example/example/server/v8/channels/api4" timestamp="2024-10-24T12:07:54Z">
<properties>
<property name="go.version" value="go1.22.6 linux/amd64"></property>
</properties>
<testcase classname="github.com/example/example/server/v8/channels/api4" name="TestWebSocketReconnectRace" time="3.320000">
<failure message="Failed" type="">=== RUN TestWebSocketReconnectRace --- FAIL: TestWebSocketReconnectRace (3.32s)</failure>
</testcase>
<testcase classname="github.com/example/example/server/v8/channels/api4" name="TestCreateChannelBookmark" time="0.000000">
<skipped message="=== RUN TestCreateChannelBookmark --- SKIP: TestCreateChannelBookmark (0.00s)"></skipped>
</testcase>
<testcase classname="github.com/example/example/server/v8/channels/api4" name="TestWebSocketUpgrade" time="2.170000"></testcase>
<testcase classname="github.com/example/example/server/v8/channels/api4" name="TestWebSocketReconnectRace" time="7.780000"></testcase>
</testsuite>
</testsuites>
Loading