From 0c63446ae535671b67a253ac1c1d43860ebeb653 Mon Sep 17 00:00:00 2001 From: Cedric Date: Tue, 12 Dec 2023 10:52:16 +0000 Subject: [PATCH] [BCF-2760] Flakey test detection improvements (#11470) * [BCF-2760] Flakey test detection improvements * [BCF-2760] Dedupe test and subtest entries --- tools/flakeytests/reporter.go | 69 +++++++-- tools/flakeytests/reporter_test.go | 103 ++++++++++--- tools/flakeytests/runner.go | 222 ++++++++++++++++++++++------- tools/flakeytests/runner_test.go | 106 +++++++++++--- 4 files changed, 399 insertions(+), 101 deletions(-) diff --git a/tools/flakeytests/reporter.go b/tools/flakeytests/reporter.go index 6696ec29a40..f17a44ef9f1 100644 --- a/tools/flakeytests/reporter.go +++ b/tools/flakeytests/reporter.go @@ -11,6 +11,12 @@ import ( "time" ) +const ( + messageType_flakeyTest = "flakey_test" + messageType_runReport = "run_report" + messageType_packagePanic = "package_panic" +) + type pushRequest struct { Streams []stream `json:"streams"` } @@ -20,16 +26,28 @@ type stream struct { Values [][]string `json:"values"` } +type BaseMessage struct { + MessageType string `json:"message_type"` + Context +} + type flakeyTest struct { + BaseMessage Package string `json:"package"` TestName string `json:"test_name"` FQTestName string `json:"fq_test_name"` - Context } -type numFlakes struct { - NumFlakes int `json:"num_flakes"` - Context +type packagePanic struct { + BaseMessage + Package string `json:"package"` +} + +type runReport struct { + BaseMessage + NumPackagePanics int `json:"num_package_panics"` + NumFlakes int `json:"num_flakes"` + NumCombined int `json:"num_combined"` } type Context struct { @@ -48,17 +66,21 @@ type LokiReporter struct { ctx Context } -func (l *LokiReporter) createRequest(flakeyTests map[string]map[string]struct{}) (pushRequest, error) { +func (l *LokiReporter) createRequest(report *Report) (pushRequest, error) { vs := [][]string{} now := l.now() nows := fmt.Sprintf("%d", now.UnixNano()) - for pkg, tests := range flakeyTests { + + for pkg, tests := range report.tests { for t := range tests { d, err := json.Marshal(flakeyTest{ + BaseMessage: BaseMessage{ + MessageType: messageType_flakeyTest, + Context: l.ctx, + }, Package: pkg, TestName: t, FQTestName: fmt.Sprintf("%s:%s", pkg, t), - Context: l.ctx, }) if err != nil { return pushRequest{}, err @@ -67,10 +89,35 @@ func (l *LokiReporter) createRequest(flakeyTests map[string]map[string]struct{}) } } - // Flakes are store in a map[string][]string, so to count them, we can't just do len(flakeyTests), + // Flakes are stored in a map[string][]string, so to count them, we can't just do len(flakeyTests), // as that will get us the number of flakey packages, not the number of flakes tests. // However, we do emit one log line per flakey test above, so use that to count our flakes. - f, err := json.Marshal(numFlakes{NumFlakes: len(vs), Context: l.ctx}) + numFlakes := len(vs) + + for pkg := range report.packagePanics { + d, err := json.Marshal(packagePanic{ + BaseMessage: BaseMessage{ + MessageType: messageType_packagePanic, + Context: l.ctx, + }, + Package: pkg, + }) + if err != nil { + return pushRequest{}, err + } + + vs = append(vs, []string{nows, string(d)}) + } + + f, err := json.Marshal(runReport{ + BaseMessage: BaseMessage{ + MessageType: messageType_runReport, + Context: l.ctx, + }, + NumFlakes: numFlakes, + NumPackagePanics: len(report.packagePanics), + NumCombined: numFlakes + len(report.packagePanics), + }) if err != nil { return pushRequest{}, nil } @@ -120,8 +167,8 @@ func (l *LokiReporter) makeRequest(pushReq pushRequest) error { return err } -func (l *LokiReporter) Report(flakeyTests map[string]map[string]struct{}) error { - pushReq, err := l.createRequest(flakeyTests) +func (l *LokiReporter) Report(report *Report) error { + pushReq, err := l.createRequest(report) if err != nil { return err } diff --git a/tools/flakeytests/reporter_test.go b/tools/flakeytests/reporter_test.go index 9cb2c8e9f7d..15650fc7bd1 100644 --- a/tools/flakeytests/reporter_test.go +++ b/tools/flakeytests/reporter_test.go @@ -12,68 +12,131 @@ import ( func TestMakeRequest_SingleTest(t *testing.T) { now := time.Now() ts := fmt.Sprintf("%d", now.UnixNano()) - ft := map[string]map[string]struct{}{ - "core/assets": map[string]struct{}{ - "TestLink": struct{}{}, + r := &Report{ + tests: map[string]map[string]int{ + "core/assets": map[string]int{ + "TestLink": 1, + }, }, } lr := &LokiReporter{auth: "bla", host: "bla", command: "go_core_tests", now: func() time.Time { return now }} - pr, err := lr.createRequest(ft) + pr, err := lr.createRequest(r) require.NoError(t, err) assert.Len(t, pr.Streams, 1) assert.Equal(t, pr.Streams[0].Stream, map[string]string{"command": "go_core_tests", "app": "flakey-test-reporter"}) assert.ElementsMatch(t, pr.Streams[0].Values, [][]string{ - {ts, `{"package":"core/assets","test_name":"TestLink","fq_test_name":"core/assets:TestLink","commit_sha":"","repository":"","event_type":""}`}, - {ts, `{"num_flakes":1,"commit_sha":"","repository":"","event_type":""}`}, + {ts, `{"message_type":"flakey_test","commit_sha":"","repository":"","event_type":"","package":"core/assets","test_name":"TestLink","fq_test_name":"core/assets:TestLink"}`}, + {ts, `{"message_type":"run_report","commit_sha":"","repository":"","event_type":"","num_package_panics":0,"num_flakes":1,"num_combined":1}`}, }) } func TestMakeRequest_MultipleTests(t *testing.T) { now := time.Now() ts := fmt.Sprintf("%d", now.UnixNano()) - ft := map[string]map[string]struct{}{ - "core/assets": map[string]struct{}{ - "TestLink": struct{}{}, - "TestCore": struct{}{}, + r := &Report{ + tests: map[string]map[string]int{ + "core/assets": map[string]int{ + "TestLink": 1, + "TestCore": 1, + }, }, } lr := &LokiReporter{auth: "bla", host: "bla", command: "go_core_tests", now: func() time.Time { return now }} - pr, err := lr.createRequest(ft) + pr, err := lr.createRequest(r) require.NoError(t, err) assert.Len(t, pr.Streams, 1) assert.Equal(t, pr.Streams[0].Stream, map[string]string{"command": "go_core_tests", "app": "flakey-test-reporter"}) assert.ElementsMatch(t, pr.Streams[0].Values, [][]string{ - {ts, `{"package":"core/assets","test_name":"TestLink","fq_test_name":"core/assets:TestLink","commit_sha":"","repository":"","event_type":""}`}, - {ts, `{"package":"core/assets","test_name":"TestCore","fq_test_name":"core/assets:TestCore","commit_sha":"","repository":"","event_type":""}`}, - {ts, `{"num_flakes":2,"commit_sha":"","repository":"","event_type":""}`}, + {ts, `{"message_type":"flakey_test","commit_sha":"","repository":"","event_type":"","package":"core/assets","test_name":"TestLink","fq_test_name":"core/assets:TestLink"}`}, + {ts, `{"message_type":"flakey_test","commit_sha":"","repository":"","event_type":"","package":"core/assets","test_name":"TestCore","fq_test_name":"core/assets:TestCore"}`}, + {ts, `{"message_type":"run_report","commit_sha":"","repository":"","event_type":"","num_package_panics":0,"num_flakes":2,"num_combined":2}`}, }) } func TestMakeRequest_NoTests(t *testing.T) { now := time.Now() ts := fmt.Sprintf("%d", now.UnixNano()) - ft := map[string]map[string]struct{}{} + r := NewReport() lr := &LokiReporter{auth: "bla", host: "bla", command: "go_core_tests", now: func() time.Time { return now }} - pr, err := lr.createRequest(ft) + pr, err := lr.createRequest(r) require.NoError(t, err) assert.Len(t, pr.Streams, 1) assert.Equal(t, pr.Streams[0].Stream, map[string]string{"command": "go_core_tests", "app": "flakey-test-reporter"}) assert.ElementsMatch(t, pr.Streams[0].Values, [][]string{ - {ts, `{"num_flakes":0,"commit_sha":"","repository":"","event_type":""}`}, + {ts, `{"message_type":"run_report","commit_sha":"","repository":"","event_type":"","num_package_panics":0,"num_flakes":0,"num_combined":0}`}, }) } func TestMakeRequest_WithContext(t *testing.T) { now := time.Now() ts := fmt.Sprintf("%d", now.UnixNano()) - ft := map[string]map[string]struct{}{} + r := NewReport() lr := &LokiReporter{auth: "bla", host: "bla", command: "go_core_tests", now: func() time.Time { return now }, ctx: Context{CommitSHA: "42"}} - pr, err := lr.createRequest(ft) + pr, err := lr.createRequest(r) require.NoError(t, err) assert.Len(t, pr.Streams, 1) assert.Equal(t, pr.Streams[0].Stream, map[string]string{"command": "go_core_tests", "app": "flakey-test-reporter"}) assert.ElementsMatch(t, pr.Streams[0].Values, [][]string{ - {ts, `{"num_flakes":0,"commit_sha":"42","repository":"","event_type":""}`}, + {ts, `{"message_type":"run_report","commit_sha":"42","repository":"","event_type":"","num_package_panics":0,"num_flakes":0,"num_combined":0}`}, }) } + +func TestMakeRequest_Panics(t *testing.T) { + now := time.Now() + ts := fmt.Sprintf("%d", now.UnixNano()) + r := &Report{ + tests: map[string]map[string]int{ + "core/assets": map[string]int{ + "TestLink": 1, + }, + }, + packagePanics: map[string]int{ + "core/assets": 1, + }, + } + lr := &LokiReporter{auth: "bla", host: "bla", command: "go_core_tests", now: func() time.Time { return now }} + pr, err := lr.createRequest(r) + require.NoError(t, err) + assert.Len(t, pr.Streams, 1) + assert.Equal(t, pr.Streams[0].Stream, map[string]string{"command": "go_core_tests", "app": "flakey-test-reporter"}) + + assert.ElementsMatch(t, pr.Streams[0].Values, [][]string{ + {ts, `{"message_type":"flakey_test","commit_sha":"","repository":"","event_type":"","package":"core/assets","test_name":"TestLink","fq_test_name":"core/assets:TestLink"}`}, + {ts, `{"message_type":"package_panic","commit_sha":"","repository":"","event_type":"","package":"core/assets"}`}, + {ts, `{"message_type":"run_report","commit_sha":"","repository":"","event_type":"","num_package_panics":1,"num_flakes":1,"num_combined":2}`}, + }) +} + +func TestDedupeEntries(t *testing.T) { + r := &Report{ + tests: map[string]map[string]int{ + "core/assets": map[string]int{ + "TestSomethingAboutAssets/test_1": 2, + "TestSomethingAboutAssets": 4, + "TestSomeOtherTest": 1, + "TestSomethingAboutAssets/test_2": 2, + "TestFinalTest/test_1": 1, + }, + "core/services/important_service": map[string]int{ + "TestAnImportantService/a_subtest": 1, + }, + }, + } + + otherReport, err := dedupeEntries(r) + require.NoError(t, err) + + expectedMap := map[string]map[string]int{ + "core/assets": map[string]int{ + "TestSomethingAboutAssets/test_1": 2, + "TestSomeOtherTest": 1, + "TestSomethingAboutAssets/test_2": 2, + "TestFinalTest/test_1": 1, + }, + "core/services/important_service": map[string]int{ + "TestAnImportantService/a_subtest": 1, + }, + } + assert.Equal(t, expectedMap, otherReport.tests) +} diff --git a/tools/flakeytests/runner.go b/tools/flakeytests/runner.go index d935000222f..97402633f38 100644 --- a/tools/flakeytests/runner.go +++ b/tools/flakeytests/runner.go @@ -11,6 +11,7 @@ import ( "os" "os/exec" "regexp" + "sort" "strings" "time" ) @@ -32,10 +33,10 @@ type tester interface { } type reporter interface { - Report(map[string]map[string]struct{}) error + Report(r *Report) error } -type parseFn func(readers ...io.Reader) (map[string]map[string]int, error) +type parseFn func(readers ...io.Reader) (*Report, error) func NewRunner(readers []io.Reader, reporter reporter, numReruns int) *Runner { tc := &testCommand{ @@ -60,9 +61,14 @@ type testCommand struct { func (t *testCommand) test(pkg string, tests []string, w io.Writer) error { replacedPkg := strings.Replace(pkg, t.repo, "", -1) - testFilter := strings.Join(tests, "|") cmd := exec.Command(t.command, fmt.Sprintf(".%s", replacedPkg)) //#nosec - cmd.Env = append(os.Environ(), fmt.Sprintf("TEST_FLAGS=-run %s", testFilter)) + cmd.Env = os.Environ() + + if len(tests) > 0 { + testFilter := strings.Join(tests, "|") + cmd.Env = append(cmd.Env, fmt.Sprintf("TEST_FLAGS=-run %s", testFilter)) + } + cmd.Stdout = io.MultiWriter(os.Stdout, w) cmd.Stderr = io.MultiWriter(os.Stderr, w) t.overrides(cmd) @@ -84,8 +90,8 @@ func newEvent(b []byte) (*TestEvent, error) { return e, err } -func parseOutput(readers ...io.Reader) (map[string]map[string]int, error) { - tests := map[string]map[string]int{} +func parseOutput(readers ...io.Reader) (*Report, error) { + report := NewReport() for _, r := range readers { s := bufio.NewScanner(r) for s.Scan() { @@ -105,24 +111,32 @@ func parseOutput(readers ...io.Reader) (map[string]map[string]int, error) { return nil, err } - // We're only interested in test failures, for which - // both Package and Test would be present. - if e.Package == "" || e.Test == "" { - continue - } - switch e.Action { case "fail": - if tests[e.Package] == nil { - tests[e.Package] = map[string]int{} + // Fail logs come in two forms: + // - with e.Package && e.Test, in which case it indicates a test failure. + // - with e.Package only, which indicates that the package test has failed, + // or possible that there has been a panic in an out-of-process goroutine running + // as part of the tests. + // + // We can ignore the last case because a package failure will be accounted elsewhere, either + // in the form of a failing test entry, or in the form of a panic output log, covered below. + if e.Test == "" { + continue } - tests[e.Package][e.Test]++ + + report.IncTest(e.Package, e.Test) case "output": if panicRe.MatchString(e.Output) { - if tests[e.Package] == nil { - tests[e.Package] = map[string]int{} + // Similar to the above, a panic can come in two forms: + // - attached to a test (i.e. with e.Test != ""), in which case + // we'll treat it like a failing test. + // - package-scoped, in which case we'll treat it as a package panic. + if e.Test != "" { + report.IncTest(e.Package, e.Test) + } else { + report.IncPackagePanic(e.Package) } - tests[e.Package][e.Test]++ } } } @@ -131,75 +145,183 @@ func parseOutput(readers ...io.Reader) (map[string]map[string]int, error) { return nil, err } } - return tests, nil + + return report, nil } type exitCoder interface { ExitCode() int } -func (r *Runner) runTests(failedTests map[string]map[string]int) (map[string]map[string]struct{}, error) { - suspectedFlakes := map[string]map[string]struct{}{} +type Report struct { + tests map[string]map[string]int + packagePanics map[string]int +} + +func NewReport() *Report { + return &Report{ + tests: map[string]map[string]int{}, + packagePanics: map[string]int{}, + } +} + +func (r *Report) HasFlakes() bool { + return len(r.tests) > 0 || len(r.packagePanics) > 0 +} + +func (r *Report) SetTest(pkg, test string, val int) { + if r.tests[pkg] == nil { + r.tests[pkg] = map[string]int{} + } + r.tests[pkg][test] = val +} + +func (r *Report) IncTest(pkg string, test string) { + if r.tests[pkg] == nil { + r.tests[pkg] = map[string]int{} + } + r.tests[pkg][test]++ +} - for pkg, tests := range failedTests { +func (r *Report) IncPackagePanic(pkg string) { + r.packagePanics[pkg]++ +} + +func (r *Runner) runTest(pkg string, tests []string) (*Report, error) { + var out bytes.Buffer + err := r.testCommand.test(pkg, tests, &out) + if err != nil { + log.Printf("Test command errored: %s\n", err) + // There was an error because the command failed with a non-zero + // exit code. This could just mean that the test failed again, so let's + // keep going. + var exErr exitCoder + if errors.As(err, &exErr) && exErr.ExitCode() > 0 { + return r.parse(&out) + } + return nil, err + } + + return r.parse(&out) +} + +func (r *Runner) runTests(rep *Report) (*Report, error) { + report := NewReport() + + // We need to deal with two types of flakes here: + // - flakes where we know the test that failed; in this case, we just rerun the failing test in question + // - flakes where we don't know what test failed. These are flakes where a panic occurred in an out-of-process goroutine, + // thus failing the package as a whole. For these, we'll rerun the whole package again. + for pkg, tests := range rep.tests { ts := []string{} for test := range tests { ts = append(ts, test) } - log.Printf("Executing test command with parameters: pkg=%s, tests=%+v, numReruns=%d\n", pkg, ts, r.numReruns) + log.Printf("[FLAKEY_TEST] Executing test command with parameters: pkg=%s, tests=%+v, numReruns=%d\n", pkg, ts, r.numReruns) for i := 0; i < r.numReruns; i++ { - var out bytes.Buffer - - err := r.testCommand.test(pkg, ts, &out) + pr, err := r.runTest(pkg, ts) if err != nil { - log.Printf("Test command errored: %s\n", err) - // There was an error because the command failed with a non-zero - // exit code. This could just mean that the test failed again, so let's - // keep going. - var exErr exitCoder - if errors.As(err, &exErr) && exErr.ExitCode() > 0 { - continue + return report, err + } + + for t := range tests { + failures := pr.tests[pkg][t] + if failures == 0 { + report.SetTest(pkg, t, 1) } - return suspectedFlakes, err } - fr, err := r.parse(&out) + } + } + + for pkg := range rep.packagePanics { + log.Printf("[PACKAGE_PANIC]: Executing test command with parameters: pkg=%s, numReruns=%d\n", pkg, r.numReruns) + for i := 0; i < r.numReruns; i++ { + pr, err := r.runTest(pkg, []string{}) if err != nil { - return nil, err + return report, err } - for t := range tests { - failures := fr[pkg][t] - if failures == 0 { - if suspectedFlakes[pkg] == nil { - suspectedFlakes[pkg] = map[string]struct{}{} - } - suspectedFlakes[pkg][t] = struct{}{} - } + if pr.packagePanics[pkg] == 0 { + report.IncPackagePanic(pkg) + } + } + } + + return report, nil +} + +func isSubtest(tn string) bool { + return strings.Contains(tn, "/") +} + +func isSubtestOf(st, mt string) bool { + return isSubtest(st) && strings.Contains(st, mt) +} + +func dedupeEntries(report *Report) (*Report, error) { + out := NewReport() + out.packagePanics = report.packagePanics + for pkg, tests := range report.tests { + // Sort the test names + testNames := make([]string, 0, len(tests)) + for t := range tests { + testNames = append(testNames, t) + } + + sort.Strings(testNames) + + for i, tn := range testNames { + // Is this the last element? If it is, then add it to the deduped set. + // This is because a) it's a main test, in which case we add it because + // it has no subtests following it, or b) it's a subtest, which we always add. + if i == len(testNames)-1 { + out.SetTest(pkg, tn, report.tests[pkg][tn]) + continue } + + // Next, let's compare the current item to the next one in the alphabetical order. + // In all cases we want to add the current item, UNLESS the current item is a main test, + // and the following one is a subtest of the current item. + nextItem := testNames[i+1] + if !isSubtest(tn) && isSubtestOf(nextItem, tn) { + continue + } + + out.SetTest(pkg, tn, report.tests[pkg][tn]) } + } - return suspectedFlakes, nil + return out, nil } func (r *Runner) Run() error { - failedTests, err := r.parse(r.readers...) + parseReport, err := r.parse(r.readers...) if err != nil { return err } - suspectedFlakes, err := r.runTests(failedTests) + report, err := r.runTests(parseReport) if err != nil { return err } - if len(suspectedFlakes) > 0 { - log.Printf("ERROR: Suspected flakes found: %+v\n", suspectedFlakes) + if report.HasFlakes() { + log.Printf("ERROR: Suspected flakes found: %+v\n", report) } else { log.Print("SUCCESS: No suspected flakes detected") } - return r.reporter.Report(suspectedFlakes) + // Before reporting the errors, let's dedupe some entries: + // In actuality, a failing subtest will produce two failing test entries, + // namely one for the test as a whole, and one for the subtest. + // This leads to inaccurate metrics since a failing subtest is double-counted. + report, err = dedupeEntries(report) + if err != nil { + return err + } + + return r.reporter.Report(report) } diff --git a/tools/flakeytests/runner_test.go b/tools/flakeytests/runner_test.go index 31f300dcbee..64e2a6c968a 100644 --- a/tools/flakeytests/runner_test.go +++ b/tools/flakeytests/runner_test.go @@ -12,16 +12,16 @@ import ( ) type mockReporter struct { - entries map[string]map[string]struct{} + report *Report } -func (m *mockReporter) Report(entries map[string]map[string]struct{}) error { - m.entries = entries +func (m *mockReporter) Report(report *Report) error { + m.report = report return nil } func newMockReporter() *mockReporter { - return &mockReporter{entries: map[string]map[string]struct{}{}} + return &mockReporter{report: NewReport()} } func TestParser(t *testing.T) { @@ -29,9 +29,10 @@ func TestParser(t *testing.T) { ` r := strings.NewReader(output) - ts, err := parseOutput(r) + pr, err := parseOutput(r) require.NoError(t, err) + ts := pr.tests assert.Len(t, ts, 1) assert.Len(t, ts["github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets"], 1) assert.Equal(t, ts["github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets"]["TestLink"], 1) @@ -44,9 +45,10 @@ func TestParser_SkipsNonJSON(t *testing.T) { ` r := strings.NewReader(output) - ts, err := parseOutput(r) + pr, err := parseOutput(r) require.NoError(t, err) + ts := pr.tests assert.Len(t, ts, 1) assert.Len(t, ts["github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets"], 1) assert.Equal(t, ts["github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets"]["TestLink"], 1) @@ -58,9 +60,10 @@ func TestParser_PanicDueToLogging(t *testing.T) { ` r := strings.NewReader(output) - ts, err := parseOutput(r) + pr, err := parseOutput(r) require.NoError(t, err) + ts := pr.tests assert.Len(t, ts, 1) assert.Len(t, ts["github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets"], 1) assert.Equal(t, ts["github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets"]["TestAssets_LinkScanValue"], 1) @@ -85,7 +88,7 @@ func TestParser_SuccessfulOutput(t *testing.T) { r := strings.NewReader(output) ts, err := parseOutput(r) require.NoError(t, err) - assert.Len(t, ts, 0) + assert.Len(t, ts.tests, 0) } type testAdapter func(string, []string, io.Writer) error @@ -119,8 +122,8 @@ func TestRunner_WithFlake(t *testing.T) { // to only report one failure (not two as expected). err := r.Run() require.NoError(t, err) - assert.Len(t, m.entries, 1) - _, ok := m.entries["github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets"]["TestLink"] + assert.Len(t, m.report.tests, 1) + _, ok := m.report.tests["github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets"]["TestLink"] assert.True(t, ok) } @@ -154,8 +157,8 @@ func TestRunner_WithFailedPackage(t *testing.T) { // to only report one failure (not two as expected). err := r.Run() require.NoError(t, err) - assert.Len(t, m.entries, 1) - _, ok := m.entries["github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets"]["TestLink"] + assert.Len(t, m.report.tests, 1) + _, ok := m.report.tests["github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets"]["TestLink"] assert.True(t, ok) } @@ -180,7 +183,7 @@ func TestRunner_AllFailures(t *testing.T) { err := r.Run() require.NoError(t, err) - assert.Len(t, m.entries, 0) + assert.Len(t, m.report.tests, 0) } func TestRunner_RerunSuccessful(t *testing.T) { @@ -206,7 +209,7 @@ func TestRunner_RerunSuccessful(t *testing.T) { err := r.Run() require.NoError(t, err) - _, ok := m.entries["github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets"]["TestLink"] + _, ok := m.report.tests["github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets"]["TestLink"] assert.True(t, ok) } @@ -228,7 +231,7 @@ func TestRunner_RootLevelTest(t *testing.T) { err := r.Run() require.NoError(t, err) - _, ok := m.entries["github.com/smartcontractkit/chainlink/v2/"]["TestConfigDocs"] + _, ok := m.report.tests["github.com/smartcontractkit/chainlink/v2/"]["TestConfigDocs"] assert.True(t, ok) } @@ -255,7 +258,7 @@ func TestRunner_RerunFailsWithNonzeroExitCode(t *testing.T) { err := r.Run() require.NoError(t, err) - _, ok := m.entries["github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets"]["TestLink"] + _, ok := m.report.tests["github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets"]["TestLink"] assert.True(t, ok) } @@ -295,10 +298,25 @@ func TestRunner_RerunWithNonZeroExitCodeDoesntStopCommand(t *testing.T) { calls := index assert.Equal(t, 4, calls) - _, ok := m.entries["github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets"]["TestLink"] + _, ok := m.report.tests["github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets"]["TestLink"] assert.True(t, ok) } +// Used for integration tests +func TestSkippedForTests_Subtests(t *testing.T) { + if os.Getenv("FLAKEY_TEST_RUNNER_RUN_FIXTURE_TEST") != "1" { + t.Skip() + } + + t.Run("1: should fail", func(t *testing.T) { + assert.False(t, true) + }) + + t.Run("2: should fail", func(t *testing.T) { + assert.False(t, true) + }) +} + // Used for integration tests func TestSkippedForTests(t *testing.T) { if os.Getenv("FLAKEY_TEST_RUNNER_RUN_FIXTURE_TEST") != "1" { @@ -319,7 +337,51 @@ func TestSkippedForTests_Success(t *testing.T) { assert.True(t, true) } -func TestParsesPanicCorrectly(t *testing.T) { +func TestIntegration_DealsWithSubtests(t *testing.T) { + if testing.Short() { + t.Skip() + } + + output := ` +{"Time":"2023-09-07T15:39:46.378315+01:00","Action":"fail","Package":"github.com/smartcontractkit/chainlink/v2/tools/flakeytests/","Test":"TestSkippedForTests_Subtests/1:_should_fail","Elapsed":0} +{"Time":"2023-09-07T15:39:46.378315+01:00","Action":"fail","Package":"github.com/smartcontractkit/chainlink/v2/tools/flakeytests/","Test":"TestSkippedForTests_Subtests","Elapsed":0} +{"Time":"2023-09-07T15:39:46.378315+01:00","Action":"fail","Package":"github.com/smartcontractkit/chainlink/v2/tools/flakeytests/","Test":"TestSkippedForTests_Subtests/2:_should_fail","Elapsed":0} +` + + m := newMockReporter() + tc := &testCommand{ + repo: "github.com/smartcontractkit/chainlink/v2/tools/flakeytests", + command: "../bin/go_core_tests", + overrides: func(cmd *exec.Cmd) { + cmd.Env = append(cmd.Env, "FLAKEY_TESTRUNNER_RUN_FIXTURE_TEST=1") + cmd.Stdout = io.Discard + cmd.Stderr = io.Discard + }, + } + r := &Runner{ + numReruns: 2, + readers: []io.Reader{strings.NewReader(output)}, + testCommand: tc, + parse: parseOutput, + reporter: m, + } + + err := r.Run() + require.NoError(t, err) + expectedTests := map[string]map[string]int{ + "github.com/smartcontractkit/chainlink/v2/tools/flakeytests/": { + "TestSkippedForTests_Subtests/1:_should_fail": 1, + "TestSkippedForTests_Subtests/2:_should_fail": 1, + }, + } + assert.Equal(t, expectedTests, m.report.tests) +} + +func TestIntegration_ParsesPanics(t *testing.T) { + if testing.Short() { + t.Skip() + } + output := `{"Time":"2023-09-07T15:39:46.378315+01:00","Action":"fail","Package":"github.com/smartcontractkit/chainlink/v2/tools/flakeytests/","Test":"TestSkippedForTests","Elapsed":0}` m := newMockReporter() @@ -342,11 +404,15 @@ func TestParsesPanicCorrectly(t *testing.T) { err := r.Run() require.NoError(t, err) - _, ok := m.entries["github.com/smartcontractkit/chainlink/v2/tools/flakeytests"]["TestSkippedForTests"] + _, ok := m.report.tests["github.com/smartcontractkit/chainlink/v2/tools/flakeytests"]["TestSkippedForTests"] assert.False(t, ok) } func TestIntegration(t *testing.T) { + if testing.Short() { + t.Skip() + } + output := `{"Time":"2023-09-07T15:39:46.378315+01:00","Action":"fail","Package":"github.com/smartcontractkit/chainlink/v2/tools/flakeytests/","Test":"TestSkippedForTests_Success","Elapsed":0}` m := newMockReporter() @@ -369,6 +435,6 @@ func TestIntegration(t *testing.T) { err := r.Run() require.NoError(t, err) - _, ok := m.entries["github.com/smartcontractkit/chainlink/v2/tools/flakeytests"]["TestSkippedForTests_Success"] + _, ok := m.report.tests["github.com/smartcontractkit/chainlink/v2/tools/flakeytests"]["TestSkippedForTests_Success"] assert.False(t, ok) }