diff --git a/launchable/test_runners/flutter.py b/launchable/test_runners/flutter.py index 1b7aa656..1ea59081 100644 --- a/launchable/test_runners/flutter.py +++ b/launchable/test_runners/flutter.py @@ -122,6 +122,8 @@ def _events(self) -> List: return events def parse_func(self, report_file: str) -> Generator[CaseEvent, None, None]: + # TODO: Support cases that include information about `flutter pub get` + # see detail: https://github.com/launchableinc/examples/actions/runs/11884312142/job/33112309450 with open(report_file, "r") as ndjson: for j in ndjson: if not j.strip(): @@ -204,6 +206,8 @@ def _parse_json(self, data: Dict): test.stderr += ("\n" if test.stderr else "") + data.get("error", "") elif data_type == "print": + # It's difficult to identify the "Retry" case because Flutter reports it with the same test ID + # So we won't handle it at the moment. test_id = data.get("testID") test = self._get_test(test_id) if test is None: diff --git a/tests/data/flutter/record_test_result.json b/tests/data/flutter/record_test_result.json new file mode 100644 index 00000000..28cab3eb --- /dev/null +++ b/tests/data/flutter/record_test_result.json @@ -0,0 +1,61 @@ +{ + "events": [ + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "test/skip_widget_test.dart" + }, + { + "type": "testcase", + "name": "Counter increments smoke skip test" + } + ], + "duration": 1.562, + "status": 2, + "stdout": "", + "stderr": "", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "test/failure_widget_test.dart" + }, + { + "type": "testcase", + "name": "Counter increments smoke failure test" + } + ], + "duration": 2.046, + "status": 0, + "stdout": "\u2550\u2550\u2561 EXCEPTION CAUGHT BY FLUTTER TEST FRAMEWORK \u255e\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\nThe following TestFailure was thrown running a test:\nExpected: exactly one matching candidate\n Actual: _TextWidgetFinder:\n Which: means none were found but one was expected\n\nWhen the exception was thrown, this was the stack:\n#4 main. (file:///Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/failure_widget_test.dart:30:5)\n\n#5 testWidgets.. (package:flutter_test/src/widget_tester.dart:189:15)\n\n#6 TestWidgetsFlutterBinding._runTestBody (package:flutter_test/src/binding.dart:1032:5)\n\n\n(elided one frame from package:stack_trace)\n\nThis was caught by the test expectation on the following line:\n file:///Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/failure_widget_test.dart line 30\nThe test description was:\n Counter increments smoke failure test\n\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550", + "stderr": "Test failed. See exception logs above.\nThe test description was: Counter increments smoke failure test", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "test/widget_test.dart" + }, + { + "type": "testcase", + "name": "Counter increments smoke test" + } + ], + "duration": 1.998, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + } + ], + "testRunner": "flutter", + "group": "", + "noBuild": false +} \ No newline at end of file diff --git a/tests/data/flutter/report.json b/tests/data/flutter/report.json new file mode 100644 index 00000000..67ef14e4 --- /dev/null +++ b/tests/data/flutter/report.json @@ -0,0 +1,29 @@ +{"protocolVersion":"0.1.1","runnerVersion":"1.25.7","pid":30535,"type":"start","time":0} +{"suite":{"id":0,"platform":"vm","path":"/Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/skip_widget_test.dart"},"type":"suite","time":0} +{"test":{"id":1,"name":"loading /Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/skip_widget_test.dart","suiteID":0,"groupIDs":[],"metadata":{"skip":false,"skipReason":null},"line":null,"column":null,"url":null},"type":"testStart","time":0} +{"suite":{"id":2,"platform":"vm","path":"/Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/failure_widget_test.dart"},"type":"suite","time":2} +{"test":{"id":3,"name":"loading /Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/failure_widget_test.dart","suiteID":2,"groupIDs":[],"metadata":{"skip":false,"skipReason":null},"line":null,"column":null,"url":null},"type":"testStart","time":2} +{"suite":{"id":4,"platform":"vm","path":"/Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/widget_test.dart"},"type":"suite","time":3} +{"test":{"id":5,"name":"loading /Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/widget_test.dart","suiteID":4,"groupIDs":[],"metadata":{"skip":false,"skipReason":null},"line":null,"column":null,"url":null},"type":"testStart","time":3} +{"count":3,"time":3,"type":"allSuites"} + +[{"event":"test.startedProcess","params":{"vmServiceUri":null,"observatoryUri":null}}] + +[{"event":"test.startedProcess","params":{"vmServiceUri":null,"observatoryUri":null}}] + +[{"event":"test.startedProcess","params":{"vmServiceUri":null,"observatoryUri":null}}] +{"testID":5,"result":"success","skipped":false,"hidden":true,"type":"testDone","time":1558} +{"group":{"id":6,"suiteID":4,"parentID":null,"name":"","metadata":{"skip":false,"skipReason":null},"testCount":1,"line":null,"column":null,"url":null},"type":"group","time":1560} +{"test":{"id":7,"name":"Counter increments smoke test","suiteID":4,"groupIDs":[6],"metadata":{"skip":false,"skipReason":null},"line":171,"column":5,"url":"package:flutter_test/src/widget_tester.dart","root_line":14,"root_column":3,"root_url":"file:///Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/widget_test.dart"},"type":"testStart","time":1560} +{"testID":3,"result":"success","skipped":false,"hidden":true,"type":"testDone","time":1561} +{"group":{"id":8,"suiteID":2,"parentID":null,"name":"","metadata":{"skip":false,"skipReason":null},"testCount":1,"line":null,"column":null,"url":null},"type":"group","time":1561} +{"test":{"id":9,"name":"Counter increments smoke failure test","suiteID":2,"groupIDs":[8],"metadata":{"skip":false,"skipReason":null},"line":171,"column":5,"url":"package:flutter_test/src/widget_tester.dart","root_line":14,"root_column":3,"root_url":"file:///Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/failure_widget_test.dart"},"type":"testStart","time":1561} +{"testID":1,"result":"success","skipped":false,"hidden":true,"type":"testDone","time":1562} +{"group":{"id":10,"suiteID":0,"parentID":null,"name":"","metadata":{"skip":false,"skipReason":null},"testCount":1,"line":null,"column":null,"url":null},"type":"group","time":1562} +{"test":{"id":11,"name":"Counter increments smoke skip test","suiteID":0,"groupIDs":[10],"metadata":{"skip":true,"skipReason":null},"line":171,"column":5,"url":"package:flutter_test/src/widget_tester.dart","root_line":14,"root_column":3,"root_url":"file:///Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/skip_widget_test.dart"},"type":"testStart","time":1562} +{"testID":11,"result":"success","skipped":true,"hidden":false,"type":"testDone","time":1562} +{"testID":7,"result":"success","skipped":false,"hidden":false,"type":"testDone","time":1998} +{"testID":9,"messageType":"print","message":"══╡ EXCEPTION CAUGHT BY FLUTTER TEST FRAMEWORK ╞════════════════════════════════════════════════════\nThe following TestFailure was thrown running a test:\nExpected: exactly one matching candidate\n Actual: _TextWidgetFinder:\n Which: means none were found but one was expected\n\nWhen the exception was thrown, this was the stack:\n#4 main. (file:///Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/failure_widget_test.dart:30:5)\n\n#5 testWidgets.. (package:flutter_test/src/widget_tester.dart:189:15)\n\n#6 TestWidgetsFlutterBinding._runTestBody (package:flutter_test/src/binding.dart:1032:5)\n\n\n(elided one frame from package:stack_trace)\n\nThis was caught by the test expectation on the following line:\n file:///Users/yabuki-ryosuke/src/github.com/launchableinc/examples/flutter/test/failure_widget_test.dart line 30\nThe test description was:\n Counter increments smoke failure test\n════════════════════════════════════════════════════════════════════════════════════════════════════","type":"print","time":2042} +{"testID":9,"error":"Test failed. See exception logs above.\nThe test description was: Counter increments smoke failure test","stackTrace":"","isFailure":false,"type":"error","time":2044} +{"testID":9,"result":"error","skipped":false,"hidden":false,"type":"testDone","time":2046} +{"success":false,"type":"done","time":2052}