From 4f9db998a5ee83b2ba8c92b3871416733b76e441 Mon Sep 17 00:00:00 2001 From: Konboi Date: Wed, 26 Apr 2023 23:29:44 +0900 Subject: [PATCH 01/16] add stdout/stderr method to parse messages --- launchable/commands/record/case_event.py | 53 +++++++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/launchable/commands/record/case_event.py b/launchable/commands/record/case_event.py index af88afc92..5c56edc02 100644 --- a/launchable/commands/record/case_event.py +++ b/launchable/commands/record/case_event.py @@ -6,6 +6,8 @@ from ...testpath import FilePathNormalizer, TestPath +POSSIBLE_RESULTS = (Failure, Error, Skipped) + CaseEventType = Dict[str, str] @@ -76,10 +78,57 @@ def path_canonicalizer(test_path: TestPath) -> TestPath: return test_path + def stdout(case: TestCase) -> str: + """ + case for: + + ... + + """ + if case.system_out is not None: + return case.system_out + + """ + case for: + + + + """ + stdout = "" + if (len(case.result) > 0): + for result in case.result: + if any(isinstance(result, r) for r in POSSIBLE_RESULTS): + stdout = stdout + result.message + + return stdout + + def stderr(case: TestCase) -> str: + """ + case for: + + ... + + """ + if case.system_err is not None: + return case.system_err + + """ + case for: + + ... + + """ + stderr = "" + for result in case.result: + if any(isinstance(result, r) for r in POSSIBLE_RESULTS): + stderr = stderr + result.text + + return stderr + return CaseEvent.create( path_canonicalizer(path_builder(case, suite, report_file)), case.time, status, - case._elem.attrib.get("system-out"), - case._elem.attrib.get("system-err"), + stdout(case), + stderr(case), suite.timestamp, data) @classmethod From 5abfca3b8c763deb686b23fab43d73f1e18b8e1e Mon Sep 17 00:00:00 2001 From: Konboi Date: Thu, 27 Apr 2023 09:29:40 +0900 Subject: [PATCH 02/16] fix format --- launchable/commands/record/case_event.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/launchable/commands/record/case_event.py b/launchable/commands/record/case_event.py index 5c56edc02..f2aaae75d 100644 --- a/launchable/commands/record/case_event.py +++ b/launchable/commands/record/case_event.py @@ -88,19 +88,7 @@ def stdout(case: TestCase) -> str: if case.system_out is not None: return case.system_out - """ - case for: - - - - """ - stdout = "" - if (len(case.result) > 0): - for result in case.result: - if any(isinstance(result, r) for r in POSSIBLE_RESULTS): - stdout = stdout + result.message - - return stdout + return "" def stderr(case: TestCase) -> str: """ @@ -115,13 +103,13 @@ def stderr(case: TestCase) -> str: """ case for: - ... + Date: Thu, 27 Apr 2023 11:28:09 +0900 Subject: [PATCH 03/16] join with \n --- launchable/commands/record/case_event.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/launchable/commands/record/case_event.py b/launchable/commands/record/case_event.py index f2aaae75d..4f5983a3b 100644 --- a/launchable/commands/record/case_event.py +++ b/launchable/commands/record/case_event.py @@ -109,7 +109,12 @@ def stderr(case: TestCase) -> str: stderr = "" for result in case.result: if any(isinstance(result, r) for r in POSSIBLE_RESULTS): - stderr = stderr + result.message + result.text + if result.message and result.text: + stderr = stderr + result.message + "\n" + result.text + elif result.message != "": + stderr = stderr + result.message + elif result.text != "": + stderr = stderr + result.text return stderr From 25571d6037d968edad6b17979a6259917c415176 Mon Sep 17 00:00:00 2001 From: Konboi Date: Thu, 27 Apr 2023 11:28:50 +0900 Subject: [PATCH 04/16] update result of ant --- tests/data/ant/record_test_result.json | 32 ++++++++++++++++++++------ 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/tests/data/ant/record_test_result.json b/tests/data/ant/record_test_result.json index c32175b76..71a9b2fe1 100644 --- a/tests/data/ant/record_test_result.json +++ b/tests/data/ant/record_test_result.json @@ -3,21 +3,33 @@ { "type": "case", "testPath": [ - { "type": "class", "name": "com.launchable.HelloWorldTest" }, - { "type": "testcase", "name": "testWillAlwaysFail" } + { + "type": "class", + "name": "com.launchable.HelloWorldTest" + }, + { + "type": "testcase", + "name": "testWillAlwaysFail" + } ], "duration": 0.006, "status": 0, "stdout": "", - "stderr": "", + "stderr": "An error message\njunit.framework.AssertionFailedError: An error message\n\tat com.launchable.HelloWorldTest.testWillAlwaysFail(Unknown Source)\n", "created_at": "2021-04-28T10:36:31", "data": null }, { "type": "case", "testPath": [ - { "type": "class", "name": "com.launchable.HelloWorldTest" }, - { "type": "testcase", "name": "testNothing" } + { + "type": "class", + "name": "com.launchable.HelloWorldTest" + }, + { + "type": "testcase", + "name": "testNothing" + } ], "duration": 0.0, "status": 1, @@ -29,8 +41,14 @@ { "type": "case", "testPath": [ - { "type": "class", "name": "com.launchable.library.CacheTest" }, - { "type": "testcase", "name": "testCache" } + { + "type": "class", + "name": "com.launchable.library.CacheTest" + }, + { + "type": "testcase", + "name": "testCache" + } ], "duration": 0.001, "status": 1, From 92be4de426eb224f7c4c0133ea94a900236e6d8b Mon Sep 17 00:00:00 2001 From: Konboi Date: Thu, 27 Apr 2023 11:49:13 +0900 Subject: [PATCH 05/16] update result of behave --- tests/data/behave/record_test_result.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data/behave/record_test_result.json b/tests/data/behave/record_test_result.json index 0b4b12290..edf37e321 100644 --- a/tests/data/behave/record_test_result.json +++ b/tests/data/behave/record_test_result.json @@ -18,7 +18,7 @@ ], "duration": 0.000202, "status": 1, - "stdout": "", + "stdout": "\n\n@scenario.begin\n Scenario: run a simple test\n Given we have behave installed ... passed in 0.000s\n When we implement a test ... passed in 0.000s\n Then behave will test it for us! ... passed in 0.000s\n\n@scenario.end\n--------------------------------------------------------------------------------\n\n", "stderr": "", "data": null } From 9ffa81ba610709f2dd1611a65cde873c8c2aa238 Mon Sep 17 00:00:00 2001 From: Konboi Date: Thu, 27 Apr 2023 11:53:57 +0900 Subject: [PATCH 06/16] update result of cucumber --- tests/data/cucumber/record_test_result.json | 52 ++++++++++----------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/tests/data/cucumber/record_test_result.json b/tests/data/cucumber/record_test_result.json index 7abf59d1b..daaa8f152 100644 --- a/tests/data/cucumber/record_test_result.json +++ b/tests/data/cucumber/record_test_result.json @@ -18,8 +18,8 @@ ], "duration": 0.000274, "status": 1, - "stdout": "", - "stderr": "", + "stdout": "\n \n ", + "stderr": "\n \n ", "data": null }, { @@ -40,8 +40,8 @@ ], "duration": 0.000241, "status": 1, - "stdout": "", - "stderr": "", + "stdout": "\n \n ", + "stderr": "\n \n ", "data": null }, { @@ -62,8 +62,8 @@ ], "duration": 0.000162, "status": 1, - "stdout": "", - "stderr": "", + "stdout": "\n \n ", + "stderr": "\n \n ", "data": null }, { @@ -84,8 +84,8 @@ ], "duration": 0.00015, "status": 1, - "stdout": "", - "stderr": "", + "stdout": "\n \n ", + "stderr": "\n \n ", "data": null }, { @@ -106,8 +106,8 @@ ], "duration": 0.000156, "status": 1, - "stdout": "", - "stderr": "", + "stdout": "\n \n ", + "stderr": "\n \n ", "data": null }, { @@ -128,8 +128,8 @@ ], "duration": 0.000309, "status": 0, - "stdout": "", - "stderr": "", + "stdout": "\n \n ", + "stderr": "\n \n ", "data": null }, { @@ -150,8 +150,8 @@ ], "duration": 0.000175, "status": 2, - "stdout": "", - "stderr": "", + "stdout": "\n \n ", + "stderr": "\n \n ", "data": null }, { @@ -172,8 +172,8 @@ ], "duration": 0.000582, "status": 1, - "stdout": "", - "stderr": "", + "stdout": "\n \n ", + "stderr": "\n \n ", "data": null }, { @@ -194,8 +194,8 @@ ], "duration": 0.000146, "status": 1, - "stdout": "", - "stderr": "", + "stdout": "\n \n ", + "stderr": "\n \n ", "data": null }, { @@ -216,8 +216,8 @@ ], "duration": 0.000138, "status": 1, - "stdout": "", - "stderr": "", + "stdout": "\n \n ", + "stderr": "\n \n ", "data": null }, { @@ -238,8 +238,8 @@ ], "duration": 0.00013, "status": 1, - "stdout": "", - "stderr": "", + "stdout": "\n \n ", + "stderr": "\n \n ", "data": null }, { @@ -260,8 +260,8 @@ ], "duration": 0.000214, "status": 1, - "stdout": "", - "stderr": "", + "stdout": "\n \n ", + "stderr": "\n \n ", "data": null }, { @@ -282,8 +282,8 @@ ], "duration": 0.011704, "status": 0, - "stdout": "", - "stderr": "", + "stdout": "\n \n ", + "stderr": "\n \n ", "data": null } ], From 8a4425af4eb8e2e4bac00d5002e7d6501e645e48 Mon Sep 17 00:00:00 2001 From: Konboi Date: Thu, 27 Apr 2023 11:55:37 +0900 Subject: [PATCH 07/16] update result of googletest --- tests/data/googletest/fail/record_test_result.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data/googletest/fail/record_test_result.json b/tests/data/googletest/fail/record_test_result.json index 3ffb1ff55..038e07bc0 100644 --- a/tests/data/googletest/fail/record_test_result.json +++ b/tests/data/googletest/fail/record_test_result.json @@ -15,7 +15,7 @@ "duration": 0.001, "status": 0, "stdout": "", - "stderr": "", + "stderr": "github.com/launchableinc/rocket-car-googletest/test_a.cpp:21\nExpected equality of these values:\n false\n true\ngithub.com/launchableinc/rocket-car-googletest/test_a.cpp:21\nExpected equality of these values:\n false\n true", "created_at": "2021-01-26T14:56:53", "data": null }, From 43899e246f28c7bed27563c1902a47acc62778c9 Mon Sep 17 00:00:00 2001 From: Konboi Date: Thu, 27 Apr 2023 11:57:34 +0900 Subject: [PATCH 08/16] update result of pytest --- tests/data/pytest/record_test_result.json | 306 +++++++++++----------- 1 file changed, 153 insertions(+), 153 deletions(-) diff --git a/tests/data/pytest/record_test_result.json b/tests/data/pytest/record_test_result.json index 6a34598a2..5a1b7d687 100644 --- a/tests/data/pytest/record_test_result.json +++ b/tests/data/pytest/record_test_result.json @@ -1,168 +1,168 @@ { - "events":[ + "events": [ { - "type":"case", - "testPath":[ - { - "type":"file", - "name":"tests/data/pytest/tests/funcs3_test.py" - }, - { - "type":"class", - "name":"tests.data.pytest.tests.funcs3_test" - }, - { - "type":"testcase", - "name":"test_func4" - } - ], - "duration":0.001, - "status":1, - "stdout":"", - "stderr":"", - "created_at":"2021-12-09T14:09:43.746900", - "data":null + "type": "case", + "testPath": [ + { + "type": "file", + "name": "tests/data/pytest/tests/funcs3_test.py" + }, + { + "type": "class", + "name": "tests.data.pytest.tests.funcs3_test" + }, + { + "type": "testcase", + "name": "test_func4" + } + ], + "duration": 0.001, + "status": 1, + "stdout": "", + "stderr": "", + "created_at": "2021-12-09T14:09:43.746900", + "data": null }, { - "type":"case", - "testPath":[ - { - "type":"file", - "name":"tests/data/pytest/tests/funcs3_test.py" - }, - { - "type":"class", - "name":"tests.data.pytest.tests.funcs3_test" - }, - { - "type":"testcase", - "name":"test_func5" - } - ], - "duration":0.001, - "status":0, - "stdout":"", - "stderr":"", - "created_at":"2021-12-09T14:09:43.746900", - "data":null + "type": "case", + "testPath": [ + { + "type": "file", + "name": "tests/data/pytest/tests/funcs3_test.py" + }, + { + "type": "class", + "name": "tests.data.pytest.tests.funcs3_test" + }, + { + "type": "testcase", + "name": "test_func5" + } + ], + "duration": 0.001, + "status": 0, + "stdout": "", + "stderr": "assert 1 == False\ndef test_func5():\n > assert 1 == False\n E assert 1 == False\n\n tests/funcs3_test.py:6: AssertionError ", + "created_at": "2021-12-09T14:09:43.746900", + "data": null }, { - "type":"case", - "testPath":[ - { - "type":"file", - "name":"tests/data/pytest/tests/test_funcs1.py" - }, - { - "type":"class", - "name":"tests.data.pytest.tests.test_funcs1" - }, - { - "type":"testcase", - "name":"test_func1" - } - ], - "duration":0.001, - "status":1, - "stdout":"", - "stderr":"", - "created_at":"2021-12-09T14:09:43.746900", - "data":null + "type": "case", + "testPath": [ + { + "type": "file", + "name": "tests/data/pytest/tests/test_funcs1.py" + }, + { + "type": "class", + "name": "tests.data.pytest.tests.test_funcs1" + }, + { + "type": "testcase", + "name": "test_func1" + } + ], + "duration": 0.001, + "status": 1, + "stdout": "", + "stderr": "", + "created_at": "2021-12-09T14:09:43.746900", + "data": null }, { - "type":"case", - "testPath":[ - { - "type":"file", - "name":"tests/data/pytest/tests/test_funcs1.py" - }, - { - "type":"class", - "name":"tests.data.pytest.tests.test_funcs1" - }, - { - "type":"testcase", - "name":"test_func2" - } - ], - "duration":0.001, - "status":0, - "stdout":"", - "stderr":"", - "created_at":"2021-12-09T14:09:43.746900", - "data":null + "type": "case", + "testPath": [ + { + "type": "file", + "name": "tests/data/pytest/tests/test_funcs1.py" + }, + { + "type": "class", + "name": "tests.data.pytest.tests.test_funcs1" + }, + { + "type": "testcase", + "name": "test_func2" + } + ], + "duration": 0.001, + "status": 0, + "stdout": "", + "stderr": "assert 1 == False\ndef test_func2():\n > assert 1 == False\n E assert 1 == False\n\n tests/test_funcs1.py:6: AssertionError ", + "created_at": "2021-12-09T14:09:43.746900", + "data": null }, { - "type":"case", - "testPath":[ - { - "type":"file", - "name":"tests/data/pytest/tests/test_funcs2.py" - }, - { - "type":"class", - "name":"tests.data.pytest.tests.test_funcs2" - }, - { - "type":"testcase", - "name":"test_func3" - } - ], - "duration":0.001, - "status":1, - "stdout":"", - "stderr":"", - "created_at":"2021-12-09T14:09:43.746900", - "data":null + "type": "case", + "testPath": [ + { + "type": "file", + "name": "tests/data/pytest/tests/test_funcs2.py" + }, + { + "type": "class", + "name": "tests.data.pytest.tests.test_funcs2" + }, + { + "type": "testcase", + "name": "test_func3" + } + ], + "duration": 0.001, + "status": 1, + "stdout": "", + "stderr": "", + "created_at": "2021-12-09T14:09:43.746900", + "data": null }, { - "type":"case", - "testPath":[ - { - "type":"file", - "name":"tests/data/pytest/tests/test_funcs2.py" - }, - { - "type":"class", - "name":"tests.data.pytest.tests.test_funcs2" - }, - { - "type":"testcase", - "name":"test_func4" - } - ], - "duration":0.001, - "status":1, - "stdout":"", - "stderr":"", - "created_at":"2021-12-09T14:09:43.746900", - "data":null + "type": "case", + "testPath": [ + { + "type": "file", + "name": "tests/data/pytest/tests/test_funcs2.py" + }, + { + "type": "class", + "name": "tests.data.pytest.tests.test_funcs2" + }, + { + "type": "testcase", + "name": "test_func4" + } + ], + "duration": 0.001, + "status": 1, + "stdout": "", + "stderr": "", + "created_at": "2021-12-09T14:09:43.746900", + "data": null }, { - "type":"case", - "testPath":[ - { - "type":"file", - "name":"tests/data/pytest/tests/fooo/func4_test.py" - }, - { - "type":"class", - "name":"tests.data.pytest.tests.fooo.func4_test" - }, - { - "type":"testcase", - "name":"test_func6" - } - ], - "duration":0.001, - "status":1, - "stdout":"", - "stderr":"", - "created_at":"2021-12-09T14:09:43.746900", - "data":null + "type": "case", + "testPath": [ + { + "type": "file", + "name": "tests/data/pytest/tests/fooo/func4_test.py" + }, + { + "type": "class", + "name": "tests.data.pytest.tests.fooo.func4_test" + }, + { + "type": "testcase", + "name": "test_func6" + } + ], + "duration": 0.001, + "status": 1, + "stdout": "", + "stderr": "", + "created_at": "2021-12-09T14:09:43.746900", + "data": null } - ], - "testRunner": "pytest", - "group": "", - "noBuild": false -} + ], + "testRunner": "pytest", + "group": "", + "noBuild": false + } From a1673bbb6ee93f7d323f182b22452b3838509508 Mon Sep 17 00:00:00 2001 From: Konboi Date: Thu, 27 Apr 2023 12:05:00 +0900 Subject: [PATCH 09/16] fix check condition --- launchable/commands/record/case_event.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/launchable/commands/record/case_event.py b/launchable/commands/record/case_event.py index 4f5983a3b..a34339df3 100644 --- a/launchable/commands/record/case_event.py +++ b/launchable/commands/record/case_event.py @@ -111,9 +111,9 @@ def stderr(case: TestCase) -> str: if any(isinstance(result, r) for r in POSSIBLE_RESULTS): if result.message and result.text: stderr = stderr + result.message + "\n" + result.text - elif result.message != "": + elif result.message: stderr = stderr + result.message - elif result.text != "": + elif result.text: stderr = stderr + result.text return stderr From 86c2eae00719e53251b3285eadcc4f4535c561cd Mon Sep 17 00:00:00 2001 From: Konboi Date: Thu, 27 Apr 2023 12:05:24 +0900 Subject: [PATCH 10/16] update result of ctest --- tests/data/ctest/record_test_result.json | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/tests/data/ctest/record_test_result.json b/tests/data/ctest/record_test_result.json index 40cfc3a3e..8356a9e4b 100644 --- a/tests/data/ctest/record_test_result.json +++ b/tests/data/ctest/record_test_result.json @@ -3,18 +3,24 @@ { "type": "case", "testPath": [ - { "type": "testcase", "name": "FooTest.Bar" } + { + "type": "testcase", + "name": "FooTest.Bar" + } ], "duration": 0.00555322, "status": 0, "stdout": "Note: Google Test filter = FooTest.Bar\n[==========] Running 1 test from 1 test suite.\n[----------] Global test environment set-up.\n[----------] 1 test from FooTest\n[ RUN ] FooTest.Bar\n/Users/ninjin/src/github.com/launchableinc/rocket-car-googletest/test_a.cpp:21: Failure\nExpected equality of these values:\n false\n true\n[ FAILED ] FooTest.Bar (0 ms)\n[----------] 1 test from FooTest (0 ms total)\n\n[----------] Global test environment tear-down\n[==========] 1 test from 1 test suite ran. (0 ms total)\n[ PASSED ] 0 tests.\n[ FAILED ] 1 test, listed below:\n[ FAILED ] FooTest.Bar\n\n 1 FAILED TEST\n", - "stderr": "", + "stderr": "Note: Google Test filter = FooTest.Bar\n[==========] Running 1 test from 1 test suite.\n[----------] Global test environment set-up.\n[----------] 1 test from FooTest\n[ RUN ] FooTest.Bar\n/Users/ninjin/src/github.com/launchableinc/rocket-car-googletest/test_a.cpp:21: Failure\nExpected equality of these values:\n false\n true\n[ FAILED ] FooTest.Bar (0 ms)\n[----------] 1 test from FooTest (0 ms total)\n\n[----------] Global test environment tear-down\n[==========] 1 test from 1 test suite ran. (0 ms total)\n[ PASSED ] 0 tests.\n[ FAILED ] 1 test, listed below:\n[ FAILED ] FooTest.Bar\n\n 1 FAILED TEST\n", "data": null }, { "type": "case", "testPath": [ - { "type": "testcase", "name": "FooTest.Baz" } + { + "type": "testcase", + "name": "FooTest.Baz" + } ], "duration": 0.00448564, "status": 1, @@ -25,7 +31,10 @@ { "type": "case", "testPath": [ - { "type": "testcase", "name": "FooTest.Foo" } + { + "type": "testcase", + "name": "FooTest.Foo" + } ], "duration": 0.00486448, "status": 1, @@ -41,10 +50,10 @@ "name": "*/ParameterizedTest.Bar/*" } ], - "duration": 0, + "duration": 0.0, "status": 2, "stdout": "Note: Google Test filter = */ParameterizedTest.Bar/*\n[==========] Running 3 tests from 1 test suite.\n[----------] Global test environment set-up.\n[----------] 3 tests from IncetanceA/ParameterizedTest\n[ RUN ] IncetanceA/ParameterizedTest.Bar/0\n[ OK ] IncetanceA/ParameterizedTest.Bar/0 (0 ms)\n[ RUN ] IncetanceA/ParameterizedTest.Bar/1\n[ OK ] IncetanceA/ParameterizedTest.Bar/1 (0 ms)\n[ RUN ] IncetanceA/ParameterizedTest.Bar/2\n[ OK ] IncetanceA/ParameterizedTest.Bar/2 (0 ms)\n[----------] 3 tests from IncetanceA/ParameterizedTest (0 ms total)\n\n[----------] Global test environment tear-down\n[==========] 3 tests from 1 test suite ran. (0 ms total)\n[ PASSED ] 3 tests.\n", - "stderr": "", + "stderr": "Note: Google Test filter = */ParameterizedTest.Bar/*\n[==========] Running 3 tests from 1 test suite.\n[----------] Global test environment set-up.\n[----------] 3 tests from IncetanceA/ParameterizedTest\n[ RUN ] IncetanceA/ParameterizedTest.Bar/0\n[ OK ] IncetanceA/ParameterizedTest.Bar/0 (0 ms)\n[ RUN ] IncetanceA/ParameterizedTest.Bar/1\n[ OK ] IncetanceA/ParameterizedTest.Bar/1 (0 ms)\n[ RUN ] IncetanceA/ParameterizedTest.Bar/2\n[ OK ] IncetanceA/ParameterizedTest.Bar/2 (0 ms)\n[----------] 3 tests from IncetanceA/ParameterizedTest (0 ms total)\n\n[----------] Global test environment tear-down\n[==========] 3 tests from 1 test suite ran. (0 ms total)\n[ PASSED ] 3 tests.\n", "data": null } ], From de266fd28dc8f1da1bcbd045157c3104538b35a7 Mon Sep 17 00:00:00 2001 From: Konboi Date: Thu, 27 Apr 2023 12:06:39 +0900 Subject: [PATCH 11/16] update result of prove --- tests/data/prove/record_test_result.json | 254 +++++++++++------------ 1 file changed, 127 insertions(+), 127 deletions(-) diff --git a/tests/data/prove/record_test_result.json b/tests/data/prove/record_test_result.json index cccaa10a5..4ef240385 100644 --- a/tests/data/prove/record_test_result.json +++ b/tests/data/prove/record_test_result.json @@ -1,133 +1,133 @@ { "events": [ - { - "type": "case", - "testPath": [ - { - "type": "file", - "name": "t/easy/01_easy.t" - }, - { - "type": "testcase", - "name": "a + b = ab" - } - ], - "duration": 0.153996229171753, - "status": 1, - "stdout": "", - "stderr": "", - "data": null - }, - { - "type": "case", - "testPath": [ - { - "type": "file", - "name": "t/easy/01_easy.t" - }, - { - "type": "testcase", - "name": "str_concat {" - } - ], - "duration": 0.00043487548828125, - "status": 1, - "stdout": "", - "stderr": "", - "data": null - }, - { - "type": "case", - "testPath": [ - { - "type": "file", - "name": "t/math/01_math.t" - }, - { - "type": "testcase", - "name": "add(1, 2) == 3" - } - ], - "duration": 0.0723011493682861, - "status": 1, - "stdout": "", - "stderr": "", - "data": null - }, - { - "type": "case", - "testPath": [ - { - "type": "file", - "name": "t/math/01_math.t" - }, - { - "type": "testcase", - "name": "add" - } - ], - "duration": 0.000227928161621094, - "status": 1, - "stdout": "", - "stderr": "", - "data": null - }, - { - "type": "case", - "testPath": [ - { - "type": "file", - "name": "t/math/01_math.t" - }, - { - "type": "testcase", - "name": "double" - } - ], - "duration": 7.91549682617188e-05, - "status": 1, - "stdout": "", - "stderr": "", - "data": null - }, - { - "type": "case", - "testPath": [ - { - "type": "file", - "name": "t/math/01_math.t" - }, - { - "type": "testcase", - "name": "double_fail" - } - ], - "duration": 0.000271081924438477, - "status": 0, - "stdout": "", - "stderr": "", - "data": null - }, - { - "type": "case", - "testPath": [ - { - "type": "file", - "name": "t/00_compile.t" - }, - { - "type": "testcase", - "name": "use Example;" - } - ], - "duration": 0.0674030780792236, - "status": 1, - "stdout": "", - "stderr": "", - "data": null - } + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "t/easy/01_easy.t" + }, + { + "type": "testcase", + "name": "a + b = ab" + } + ], + "duration": 0.153996229171753, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "t/easy/01_easy.t" + }, + { + "type": "testcase", + "name": "str_concat {" + } + ], + "duration": 0.00043487548828125, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "t/math/01_math.t" + }, + { + "type": "testcase", + "name": "add(1, 2) == 3" + } + ], + "duration": 0.0723011493682861, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "t/math/01_math.t" + }, + { + "type": "testcase", + "name": "add" + } + ], + "duration": 0.000227928161621094, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "t/math/01_math.t" + }, + { + "type": "testcase", + "name": "double" + } + ], + "duration": 7.91549682617188e-05, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "t/math/01_math.t" + }, + { + "type": "testcase", + "name": "double_fail" + } + ], + "duration": 0.000271081924438477, + "status": 0, + "stdout": "", + "stderr": "not ok 4 - double_fail\nSubtest: double_fail\n not ok 1 - fail 1 + 2\n not ok 2 - fail 3 + 2\n 1..2\n", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "t/00_compile.t" + }, + { + "type": "testcase", + "name": "use Example;" + } + ], + "duration": 0.0674030780792236, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + } ], "testRunner": "prove", "group": "", "noBuild": false -} + } From e362b3eb2e652706ee3e1079dbe313d5fa51b7e1 Mon Sep 17 00:00:00 2001 From: Konboi Date: Thu, 27 Apr 2023 12:08:00 +0900 Subject: [PATCH 12/16] update result of robot --- tests/data/robot/record_test_result.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/data/robot/record_test_result.json b/tests/data/robot/record_test_result.json index e023c251e..16e63c1fb 100644 --- a/tests/data/robot/record_test_result.json +++ b/tests/data/robot/record_test_result.json @@ -15,7 +15,7 @@ "duration": 0.002, "status": 0, "stdout": "", - "stderr": "", + "stderr": "0 != -1", "data": null }, { @@ -69,7 +69,7 @@ "duration": 0.001, "status": 0, "stdout": "", - "stderr": "", + "stderr": "0 != -1", "data": null }, { From 20ebc916c0e6788d20f610eb5e2e0e0603cf06af Mon Sep 17 00:00:00 2001 From: Konboi Date: Thu, 27 Apr 2023 12:22:25 +0900 Subject: [PATCH 13/16] fix pytest plugin's json option --- launchable/test_runners/pytest.py | 71 +++++++- .../data/pytest/record_test_result_json.json | 161 ++++++++++++++++++ tests/test_runners/test_pytest.py | 3 +- 3 files changed, 233 insertions(+), 2 deletions(-) create mode 100644 tests/data/pytest/record_test_result_json.json diff --git a/launchable/test_runners/pytest.py b/launchable/test_runners/pytest.py index bd6284337..f38806720 100644 --- a/launchable/test_runners/pytest.py +++ b/launchable/test_runners/pytest.py @@ -194,9 +194,78 @@ def parse_func( elif outcome == "skipped": status = CaseEvent.TEST_SKIPPED + """example json + "longrepr": { + "reprcrash": { + "lineno": 6, + "message": "assert 1 == False", + "path": "/Users/yabuki-ryosuke/src/github.com/launchableinc/cli/tests/data/pytest/tests/test_funcs1.py" + }, + "reprtraceback": { + "extraline": null, + "reprentries": [ + { + "data": { + "lines": [ + " def test_func2():", + "> assert 1 == False", + "E assert 1 == False" + ], + "reprfileloc": { + "lineno": 6, + "message": "AssertionError", + "path": "tests/test_funcs1.py" + }, + "reprfuncargs": { + "args": [] + }, + "reprlocals": null, + "style": "long" + }, + "type": "ReprEntry" + } + ], + "style": "long" + }, + "sections": [] + } + """ + stdout = "" + stderr = "" + longrepr = data.get("longrepr", None) + if longrepr: + message = None + reprcrash = longrepr.get("reprcrash", None) + if reprcrash: + message = reprcrash.get("message", None) + + text = None + reprtraceback = longrepr.get("reprtraceback", None) + if reprtraceback: + reprentries = reprtraceback.get("reprentries", None) + if reprentries: + for r in reprentries: + d = r.get("data", None) + if d: + text = "\n".join(d.get("lines", [])) + + if message and text: + stderr = message + "\n" + text + elif message: + stderr = stderr + message + elif text: + stderr = stderr + message + test_path = _parse_pytest_nodeid(nodeid) for path in test_path: if path.get("type") == "file": path["name"] = pathlib.Path(path["name"]).as_posix() - yield CaseEvent.create(test_path, data.get("duration", 0), status, None, None, None, None) + yield CaseEvent.create( + test_path, + data.get("duration", 0), + status, + stdout, + stderr, + None, + None) diff --git a/tests/data/pytest/record_test_result_json.json b/tests/data/pytest/record_test_result_json.json new file mode 100644 index 000000000..a4d0e4ad6 --- /dev/null +++ b/tests/data/pytest/record_test_result_json.json @@ -0,0 +1,161 @@ +{ + "events": [ + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "tests/data/pytest/tests/funcs3_test.py" + }, + { + "type": "class", + "name": "tests.data.pytest.tests.funcs3_test" + }, + { + "type": "testcase", + "name": "test_func4" + } + ], + "duration": 0.001, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "tests/data/pytest/tests/funcs3_test.py" + }, + { + "type": "class", + "name": "tests.data.pytest.tests.funcs3_test" + }, + { + "type": "testcase", + "name": "test_func5" + } + ], + "duration": 0.001, + "status": 0, + "stdout": "", + "stderr": "assert 1 == False\n def test_func5():\n> assert 1 == False\nE assert 1 == False", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "tests/data/pytest/tests/test_funcs1.py" + }, + { + "type": "class", + "name": "tests.data.pytest.tests.test_funcs1" + }, + { + "type": "testcase", + "name": "test_func1" + } + ], + "duration": 0.001, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "tests/data/pytest/tests/test_funcs1.py" + }, + { + "type": "class", + "name": "tests.data.pytest.tests.test_funcs1" + }, + { + "type": "testcase", + "name": "test_func2" + } + ], + "duration": 0.001, + "status": 0, + "stdout": "", + "stderr": "assert 1 == False\n def test_func2():\n> assert 1 == False\nE assert 1 == False", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "tests/data/pytest/tests/test_funcs2.py" + }, + { + "type": "class", + "name": "tests.data.pytest.tests.test_funcs2" + }, + { + "type": "testcase", + "name": "test_func3" + } + ], + "duration": 0.001, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "tests/data/pytest/tests/test_funcs2.py" + }, + { + "type": "class", + "name": "tests.data.pytest.tests.test_funcs2" + }, + { + "type": "testcase", + "name": "test_func4" + } + ], + "duration": 0.001, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "file", + "name": "tests/data/pytest/tests/fooo/func4_test.py" + }, + { + "type": "class", + "name": "tests.data.pytest.tests.fooo.func4_test" + }, + { + "type": "testcase", + "name": "test_func6" + } + ], + "duration": 0.001, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + } + ], + "testRunner": "pytest", + "group": "", + "noBuild": false +} diff --git a/tests/test_runners/test_pytest.py b/tests/test_runners/test_pytest.py index e3b04fcd4..4dd7df541 100644 --- a/tests/test_runners/test_pytest.py +++ b/tests/test_runners/test_pytest.py @@ -13,6 +13,7 @@ class PytestTest(CliTestCase): test_files_dir = Path(__file__).parent.joinpath('../data/pytest/').resolve() result_file_path = test_files_dir.joinpath('record_test_result.json') + json_option_result_file_path = test_files_dir.joinpath('record_test_result_json.json') subset_input = '''tests/funcs3_test.py::test_func4 tests/funcs3_test.py::test_func5 tests/test_funcs1.py::test_func1 @@ -57,7 +58,7 @@ def test_record_test_with_json_option(self): self.assertEqual(result.exit_code, 0) payload = json.loads(gzip.decompress(responses.calls[1].request.body).decode()) - expected = self.load_json_from_file(self.result_file_path) + expected = self.load_json_from_file(self.json_option_result_file_path) for e in payload["events"]: e.pop("created_at", "") From b3a1f53e46640936e2be64f58e6d561f2438762c Mon Sep 17 00:00:00 2001 From: Konboi Date: Thu, 27 Apr 2023 14:24:51 +0900 Subject: [PATCH 14/16] fix typo --- launchable/test_runners/pytest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launchable/test_runners/pytest.py b/launchable/test_runners/pytest.py index f38806720..b5b8a7df8 100644 --- a/launchable/test_runners/pytest.py +++ b/launchable/test_runners/pytest.py @@ -254,7 +254,7 @@ def parse_func( elif message: stderr = stderr + message elif text: - stderr = stderr + message + stderr = stderr + text test_path = _parse_pytest_nodeid(nodeid) for path in test_path: From 3ba29143ed016c42455ce0cb48b22446b207ed74 Mon Sep 17 00:00:00 2001 From: Konboi Date: Thu, 27 Apr 2023 15:09:44 +0900 Subject: [PATCH 15/16] fix --- launchable/commands/record/case_event.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launchable/commands/record/case_event.py b/launchable/commands/record/case_event.py index a34339df3..dbeb66a55 100644 --- a/launchable/commands/record/case_event.py +++ b/launchable/commands/record/case_event.py @@ -108,7 +108,7 @@ def stderr(case: TestCase) -> str: """ stderr = "" for result in case.result: - if any(isinstance(result, r) for r in POSSIBLE_RESULTS): + if type(result) in POSSIBLE_RESULTS: if result.message and result.text: stderr = stderr + result.message + "\n" + result.text elif result.message: From 4a18a8fe41dfecee0acd1651b909c2901a564d02 Mon Sep 17 00:00:00 2001 From: Konboi Date: Thu, 27 Apr 2023 15:30:32 +0900 Subject: [PATCH 16/16] set new line to result.message --- launchable/commands/record/case_event.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/launchable/commands/record/case_event.py b/launchable/commands/record/case_event.py index dbeb66a55..017b50212 100644 --- a/launchable/commands/record/case_event.py +++ b/launchable/commands/record/case_event.py @@ -103,7 +103,7 @@ def stderr(case: TestCase) -> str: """ case for: - ... """ stderr = "" @@ -112,7 +112,7 @@ def stderr(case: TestCase) -> str: if result.message and result.text: stderr = stderr + result.message + "\n" + result.text elif result.message: - stderr = stderr + result.message + stderr = stderr + result.message + "\n" elif result.text: stderr = stderr + result.text