Merge pull request #346 from mhaoli/check_xctest_result

Check XCTest result and raise error if it failed

tidevice/_device.py

@@ -40,7 +40,7 @@
 from ._proto import *
 from ._safe_socket import *
 from ._sync import Sync
-from ._types import DeviceInfo
+from ._types import DeviceInfo, XCTestResult
 from ._usbmux import Usbmux
 from ._utils import (ProgressReader, get_app_dir, semver_compare,
                      set_socket_timeout)
@@ -1078,10 +1078,30 @@ def _show_log_message(m: DTXMessage):
                 logger.info("Test runner ready detected")
                 _start_executing()
 
+        test_results = []
+        test_results_lock = threading.Lock()
+
+        def _record_test_result_callback(m: DTXMessage):
+            result = None
+            if isinstance(m.result, (tuple, list)) and len(m.result) >= 1:
+              if isinstance(m.result[1], (tuple, list)):
+                  try:
+                    result = XCTestResult(*m.result[1])
+                  except TypeError:
+                    pass
+            if not result:
+                logger.warning('Ignore unknown test result message: %s', m)
+                return
+            with test_results_lock:
+                test_results.append(result)
+
         x2.register_callback(
             '_XCT_testBundleReadyWithProtocolVersion:minimumVersion:',
             _start_executing)  # This only happends <= iOS 13
         x2.register_callback('_XCT_logDebugMessage:', _show_log_message)
+        x2.register_callback(
+            "_XCT_testSuite:didFinishAt:runCount:withFailures:unexpected:testDuration:totalDuration:",
+            _record_test_result_callback)
 
         # index: 469
         identifier = '_IDE_initiateSessionWithIdentifier:forClient:atPath:protocolVersion:'
@@ -1151,7 +1171,15 @@ def _show_log_message(m: DTXMessage):
         # on windows threading.Event.wait can't handle ctrl-c
         while not quit_event.wait(.1):
             pass
-        logger.info("xctrunner quited")
+
+        test_result_str = "\n".join(map(str, test_results))
+        if any(result.failure_count > 0 for result in test_results):
+              raise RuntimeError(
+                  "Xcode test failed on device with test results:\n"
+                  f"{test_result_str}"
+              )
+
+        logger.info("xctrunner quited with result:\n%s", test_result_str)
 
 
 Device = BaseDevice

tidevice/_types.py

@@ -49,3 +49,40 @@ class DeviceInfo(_BaseInfo):
     udid: str = alias_field("SerialNumber")
     device_id: int = alias_field("DeviceID")
     conn_type: ConnectionType = alias_field("ConnectionType")
+
+
+@dataclass(frozen=True)
+class XCTestResult(_BaseInfo):
+    """Representing the XCTest result printed at the end of test.
+
+    At the end of an XCTest, the test process will print following information:
+
+        Test Suite 'MoblySignInTests' passed at 2023-09-03 16:35:39.214.
+                Executed 1 test, with 0 failures (0 unexpected) in 3.850 (3.864) seconds
+        Test Suite 'MoblySignInTests.xctest' passed at 2023-09-03 16:35:39.216.
+                 Executed 1 test, with 0 failures (0 unexpected) in 3.850 (3.866) seconds
+        Test Suite 'Selected tests' passed at 2023-09-03 16:35:39.217.
+                 Executed 1 test, with 0 failures (0 unexpected) in 3.850 (3.869) seconds
+    """
+
+    MESSAGE = (
+        "Test Suite '{test_suite_name}' passed at {end_time}.\n"
+        "\t Executed {run_count} test, with {failure_count} failures ({unexpected_count} unexpected) in {test_duration:.3f} ({total_duration:.3f}) seconds"
+    )
+
+    test_suite_name: str = alias_field('TestSuiteName')
+    end_time: str = alias_field('EndTime')
+    run_count: int = alias_field('RunCount')
+    failure_count: int = alias_field('FailureCount')
+    unexpected_count: int = alias_field('UnexpectedCount')
+    test_duration: float = alias_field('TestDuration')
+    total_duration: float = alias_field('TotalDuration')
+
+    def __repr__(self) -> str:
+        return self.MESSAGE.format(
+            test_suite_name=self.test_suite_name, end_time=self.end_time,
+            run_count=self.run_count, failure_count=self.failure_count,
+            unexpected_count=self.unexpected_count,
+            test_duration=self.test_duration,
+            total_duration=self.total_duration,
+        )