From 11bb74461724b678dc0a4164a13f2b93ced5cec6 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 5 Sep 2023 15:43:26 +0800 Subject: [PATCH 1/2] Check XCTest result and raise error if it failed --- tidevice/_device.py | 32 ++++++++++++++++++++++++++++++-- tidevice/_types.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/tidevice/_device.py b/tidevice/_device.py index 38b58e8..438669e 100644 --- a/tidevice/_device.py +++ b/tidevice/_device.py @@ -40,7 +40,7 @@ from ._proto import * from ._safe_socket import * from ._sync import Sync -from ._types import DeviceInfo +from ._types import DeviceInfo, XCTestResult from ._usbmux import Usbmux from ._utils import (ProgressReader, get_app_dir, semver_compare, set_socket_timeout) @@ -1072,10 +1072,30 @@ def _show_log_message(m: DTXMessage): logger.info("Test runner ready detected") _start_executing() + test_results = [] + test_results_lock = threading.Lock() + + def _record_test_result_callback(m: DTXMessage): + result = None + if isinstance(m.result, (tuple, list)) and len(m.result) >= 1: + if isinstance(m.result[1], (tuple, list)): + try: + result = XCTestResult(*m.result[1]) + except TypeError: + pass + if not result: + logger.warning('Ignore unknown test result message: %s', m) + return + with test_results_lock: + test_results.append(result) + x2.register_callback( '_XCT_testBundleReadyWithProtocolVersion:minimumVersion:', _start_executing) # This only happends <= iOS 13 x2.register_callback('_XCT_logDebugMessage:', _show_log_message) + x2.register_callback( + "_XCT_testSuite:didFinishAt:runCount:withFailures:unexpected:testDuration:totalDuration:", + _record_test_result_callback) # index: 469 identifier = '_IDE_initiateSessionWithIdentifier:forClient:atPath:protocolVersion:' @@ -1144,7 +1164,15 @@ def _show_log_message(m: DTXMessage): # on windows threading.Event.wait can't handle ctrl-c while not quit_event.wait(.1): pass - logger.info("xctrunner quited") + + test_result_str = "\n".join(map(str, test_results)) + if any(result.failure_count > 0 for result in test_results): + raise RuntimeError( + "Xcode test failed on device with test results:\n" + f"{test_result_str}" + ) + + logger.info("xctrunner quited with result:\n%s", test_result_str) Device = BaseDevice diff --git a/tidevice/_types.py b/tidevice/_types.py index 284fd38..af972f6 100644 --- a/tidevice/_types.py +++ b/tidevice/_types.py @@ -49,3 +49,40 @@ class DeviceInfo(_BaseInfo): udid: str = alias_field("SerialNumber") device_id: int = alias_field("DeviceID") conn_type: ConnectionType = alias_field("ConnectionType") + + +@dataclass(frozen=True) +class XCTestResult(_BaseInfo): + """Representing the XCTest result printed at the end of test. + + At the end of an XCTest, the test process will print following information: + + Test Suite 'MoblySignInTests' passed at 2023-09-03 16:35:39.214. + Executed 1 test, with 0 failures (0 unexpected) in 3.850 (3.864) seconds + Test Suite 'MoblySignInTests.xctest' passed at 2023-09-03 16:35:39.216. + Executed 1 test, with 0 failures (0 unexpected) in 3.850 (3.866) seconds + Test Suite 'Selected tests' passed at 2023-09-03 16:35:39.217. + Executed 1 test, with 0 failures (0 unexpected) in 3.850 (3.869) seconds + """ + + MESSAGE = ( + "Test Suite '{test_suite_name}' passed at {end_time}.\n" + "\t Executed {run_count} test, with {failure_count} failures ({unexpected_count} unexpected) in {test_duration:.3f} ({total_duration:.3f}) seconds" + ) + + test_suite_name: str = alias_field('TestSuiteName') + end_time: str = alias_field('EndTime') + run_count: int = alias_field('RunCount') + failure_count: int = alias_field('FailureCount') + unexpected_count: int = alias_field('UnexpectedCount') + test_duration: float = alias_field('TestDuration') + total_duration: float = alias_field('TotalDuration') + + def __repr__(self) -> str: + return self.MESSAGE.format( + test_suite_name=self.test_suite_name, end_time=self.end_time, + run_count=self.run_count, failure_count=self.failure_count, + unexpected_count=self.unexpected_count, + test_duration=self.test_duration, + total_duration=self.total_duration, + ) From ba22324b0545170133806ca71b19106be968b1b6 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 5 Sep 2023 23:06:56 +0800 Subject: [PATCH 2/2] update format --- tidevice/_types.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tidevice/_types.py b/tidevice/_types.py index af972f6..8a324b4 100644 --- a/tidevice/_types.py +++ b/tidevice/_types.py @@ -57,12 +57,12 @@ class XCTestResult(_BaseInfo): At the end of an XCTest, the test process will print following information: - Test Suite 'MoblySignInTests' passed at 2023-09-03 16:35:39.214. - Executed 1 test, with 0 failures (0 unexpected) in 3.850 (3.864) seconds - Test Suite 'MoblySignInTests.xctest' passed at 2023-09-03 16:35:39.216. - Executed 1 test, with 0 failures (0 unexpected) in 3.850 (3.866) seconds - Test Suite 'Selected tests' passed at 2023-09-03 16:35:39.217. - Executed 1 test, with 0 failures (0 unexpected) in 3.850 (3.869) seconds + Test Suite 'MoblySignInTests' passed at 2023-09-03 16:35:39.214. + Executed 1 test, with 0 failures (0 unexpected) in 3.850 (3.864) seconds + Test Suite 'MoblySignInTests.xctest' passed at 2023-09-03 16:35:39.216. + Executed 1 test, with 0 failures (0 unexpected) in 3.850 (3.866) seconds + Test Suite 'Selected tests' passed at 2023-09-03 16:35:39.217. + Executed 1 test, with 0 failures (0 unexpected) in 3.850 (3.869) seconds """ MESSAGE = ( @@ -79,10 +79,10 @@ class XCTestResult(_BaseInfo): total_duration: float = alias_field('TotalDuration') def __repr__(self) -> str: - return self.MESSAGE.format( - test_suite_name=self.test_suite_name, end_time=self.end_time, - run_count=self.run_count, failure_count=self.failure_count, - unexpected_count=self.unexpected_count, - test_duration=self.test_duration, - total_duration=self.total_duration, - ) + return self.MESSAGE.format( + test_suite_name=self.test_suite_name, end_time=self.end_time, + run_count=self.run_count, failure_count=self.failure_count, + unexpected_count=self.unexpected_count, + test_duration=self.test_duration, + total_duration=self.total_duration, + )