From 267fcb49b0bacf3fdefd07a2ffbb34f1541c2d82 Mon Sep 17 00:00:00 2001 From: Konboi Date: Thu, 6 Apr 2023 18:11:21 +0900 Subject: [PATCH 01/12] add cts profile --- launchable/test_runners/cts.py | 116 +++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 launchable/test_runners/cts.py diff --git a/launchable/test_runners/cts.py b/launchable/test_runners/cts.py new file mode 100644 index 000000000..95bd49001 --- /dev/null +++ b/launchable/test_runners/cts.py @@ -0,0 +1,116 @@ +from ast import Dict +from xml.etree import ElementTree as ET + +import click + +from launchable.commands.record.case_event import CaseEvent + +from . import launchable + + +def build_path(module: str, test_case: str, test: str): + return [ + {"type": "Module", "name": module}, + {"type": "TestCase", "name": test_case}, + {"type": "Test", "name": test}, + ] + + +def parse_func(p: str) -> Dict: + """ # noqa: E501 + # sample report format + success case: + + + + + + + failure case: + + + + + + android.accounts.OperationCanceledException + at android.accounts.AccountManager$AmsTask.internalGetResult(AccountManager.java:2393) + at android.accounts.AccountManager$AmsTask.getResult(AccountManager.java:2422) + at android.accounts.AccountManager$AmsTask.getResult(AccountManager.java:2337) + at android.accounts.cts.AbstractAuthenticatorTests.testFinishSessionAndStartAddAccountSessionDefaultImpl(AbstractAuthenticatorTests.java:165) + + + + + + """ + events = [] + tree = ET.parse(p) + + for module in tree.iter('Module'): + test_results = [] + total_duration = module.get("runtime") + module_name = module.get("name") + + for test_case in module.iter("TestCase"): + test_case_name = test_case.get('name') + + for test in test_case.iter("Test"): + result = test.get('result') + test_name = test.get("name") + stdout = "" + stderr = "" + + failure = test.find('Failure') + if failure: + stack_trace = failure.find("StackTrace").text if failure.find("StackTrace") is not None else "" + + stdout = failure.get("message") + stderr = stack_trace + + test_results.append({ + "test_case_name": test_case_name, + "test_name": test_name, + "result": result, + "stdout": stdout, + "stderr": stderr, + }) + + if len(test_results) == 0: + continue + + test_duration_msec_per_test = int(total_duration) / len(test_results) + + for result in test_results: + status = CaseEvent.TEST_PASSED + if result.get("result") == "fail": + status = CaseEvent.TEST_FAILED + elif result.get("result") == "ASSUMPTION_FAILURE" or result.get("result") == "IGNORED": + status = CaseEvent.TEST_SKIPPED + + events.append(CaseEvent.create( + test_path=build_path(module_name, result.get("test_case_name"), result.get("test_name")), + duration_secs=float(test_duration_msec_per_test / 1000), + status=status, + stdout=result.get("stdout"), + stderr=result.get("stderr"), + )) + + return (x for x in events) + + +@click.argument('reports', required=True, nargs=-1) +@launchable.record.tests +def record_tests(client, reports): + for r in reports: + client.report(r) + + client.parse_func = parse_func + client.run() + + +@launchable.subset +def subset(client): + + print("subset") + + return From 30a1c245e9b3fa58ecda0f6a862055c1c3993097 Mon Sep 17 00:00:00 2001 From: Konboi Date: Fri, 7 Apr 2023 14:48:50 +0900 Subject: [PATCH 02/12] support subset command --- launchable/test_runners/cts.py | 40 ++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/launchable/test_runners/cts.py b/launchable/test_runners/cts.py index 95bd49001..dcfff5ddd 100644 --- a/launchable/test_runners/cts.py +++ b/launchable/test_runners/cts.py @@ -1,9 +1,11 @@ +import sys from ast import Dict from xml.etree import ElementTree as ET import click from launchable.commands.record.case_event import CaseEvent +from launchable.testpath import TestPath from . import launchable @@ -110,7 +112,37 @@ def record_tests(client, reports): @launchable.subset def subset(client): - - print("subset") - - return + if not client.is_get_tests_from_previous_sessions: + click.echo(click.style( + "ERROR: cts profile supports only Zero Input Subsetting (ZIS). Please use `--get-tests-from-previous-sessions` option for subsetting", # noqa E501 + fg="red"), + err=True) + sys.exist(1) + + include_option = "--include-filter" + exclude_option = "--exclude-filter" + + def formatter(test_path: TestPath): + module = "" + test_case = "" + for path in test_path: + t = path.get('type', '') + n = path.get('name', '') + + if t == "Module": + module = n + elif t == "TestCase": + test_case = n + + if module == "" or test_case == "": + return + + option = include_option + if client.is_output_exclusion_rules: + option = exclude_option + + return "{option} \"{module} {test_case}\"".format(option=option, module=module, test_case=test_case) + + client.formatter = formatter + client.separator = " " + client.run() From 397e063e5de2cae9097b6422a0d7ae1f90670739 Mon Sep 17 00:00:00 2001 From: Konboi Date: Fri, 7 Apr 2023 14:51:26 +0900 Subject: [PATCH 03/12] fix --- launchable/test_runners/cts.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/launchable/test_runners/cts.py b/launchable/test_runners/cts.py index dcfff5ddd..d6cf11acb 100644 --- a/launchable/test_runners/cts.py +++ b/launchable/test_runners/cts.py @@ -1,4 +1,3 @@ -import sys from ast import Dict from xml.etree import ElementTree as ET @@ -117,7 +116,7 @@ def subset(client): "ERROR: cts profile supports only Zero Input Subsetting (ZIS). Please use `--get-tests-from-previous-sessions` option for subsetting", # noqa E501 fg="red"), err=True) - sys.exist(1) + exit(1) include_option = "--include-filter" exclude_option = "--exclude-filter" From e80f93afd52811b94bdc88e0abdf0da31839cbcc Mon Sep 17 00:00:00 2001 From: Konboi Date: Fri, 7 Apr 2023 16:08:48 +0900 Subject: [PATCH 04/12] add tests data --- tests/data/cts/record_test_result.json | 249 +++++++++++++++++++++++++ tests/data/cts/test_result.xml | 44 +++++ 2 files changed, 293 insertions(+) create mode 100644 tests/data/cts/record_test_result.json create mode 100644 tests/data/cts/test_result.xml diff --git a/tests/data/cts/record_test_result.json b/tests/data/cts/record_test_result.json new file mode 100644 index 000000000..f44535864 --- /dev/null +++ b/tests/data/cts/record_test_result.json @@ -0,0 +1,249 @@ +{ + "events": [ + { + "type": "case", + "testPath": [ + { + "type": "Module", + "name": "CtsAccessibilityServiceTestCases[instant]" + }, + { + "type": "TestCase", + "name": "android.accessibilityservice.cts.AccessibilityEmbeddedHierarchyTest" + }, + { + "type": "Test", + "name": "testEmbeddedViewHasCorrectBound" + } + ], + "duration": 1.283, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "Module", + "name": "CtsAccessibilityServiceTestCases[instant]" + }, + { + "type": "TestCase", + "name": "android.accessibilityservice.cts.AccessibilityEmbeddedHierarchyTest" + }, + { + "type": "Test", + "name": "testEmbeddedViewCanBeFound" + } + ], + "duration": 1.283, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "Module", + "name": "CtsAccessibilityServiceTestCases[instant]" + }, + { + "type": "TestCase", + "name": "android.accessibilityservice.cts.AccessibilityEmbeddedHierarchyTest" + }, + { + "type": "Test", + "name": "testEmbeddedViewHasCorrectBoundAfterHostViewMove" + } + ], + "duration": 1.283, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "Module", + "name": "CtsAccessibilityServiceTestCases[instant]" + }, + { + "type": "TestCase", + "name": "android.accessibilityservice.cts.AccessibilityEmbeddedHierarchyTest" + }, + { + "type": "Test", + "name": "testEmbeddedViewCanFindItsHostParent" + } + ], + "duration": 1.283, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "Module", + "name": "CtsAccessibilityServiceTestCases[instant]" + }, + { + "type": "TestCase", + "name": "android.accessibilityservice.cts.AccessibilityEmbeddedHierarchyTest" + }, + { + "type": "Test", + "name": "testEmbeddedViewIsInvisibleAfterMovingOutOfScreen" + } + ], + "duration": 1.283, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "Module", + "name": "CtsAccessibilityServiceTestCases[instant]" + }, + { + "type": "TestCase", + "name": "android.accessibilityservice.cts.AccessibilityDragAndDropTest" + }, + { + "type": "Test", + "name": "testStartDrag_eventSentAndActionsUpdated" + } + ], + "duration": 1.283, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "Module", + "name": "CtsAccessibilityServiceTestCases[instant]" + }, + { + "type": "TestCase", + "name": "android.accessibilityservice.cts.AccessibilityDragAndDropTest" + }, + { + "type": "Test", + "name": "testCancelDrag_eventSentAndActionsUpdated" + } + ], + "duration": 1.283, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "Module", + "name": "CtsAccessibilityServiceTestCases[instant]" + }, + { + "type": "TestCase", + "name": "android.accessibilityservice.cts.AccessibilityDragAndDropTest" + }, + { + "type": "Test", + "name": "testDrop_eventSentAndActionsUpdated" + } + ], + "duration": 1.283, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "Module", + "name": "CtsAccessibilityServiceTestCases" + }, + { + "type": "TestCase", + "name": "android.accessibilityservice.cts.AccessibilityGlobalActionsTest" + }, + { + "type": "Test", + "name": "testPerformGlobalActionBack" + } + ], + "duration": 0.01, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "Module", + "name": "CtsAccessibilityServiceTestCases" + }, + { + "type": "TestCase", + "name": "android.accessibilityservice.cts.AccessibilityGlobalActionsTest" + }, + { + "type": "Test", + "name": "testPerformGlobalActionHome" + } + ], + "duration": 0.01, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "Module", + "name": "CtsAccessibilityServiceTestCases" + }, + { + "type": "TestCase", + "name": "android.accessibilityservice.cts.AccessibilityGlobalActionsTest" + }, + { + "type": "Test", + "name": "testPerformGlobalActionNotifications" + } + ], + "duration": 0.01, + "status": 0, + "stdout": "java.lang.AssertionError: Unable to reach home screen\r", + "stderr": "\n java.lang.AssertionError: Unable to reach home screen\n at org.junit.Assert.fail(Assert.java:89)\n at\n android.accessibilityservice.cts.utils.ActivityLaunchUtils.homeScreenOrBust(ActivityLaunchUtils.java:161)\n at\n android.accessibilityservice.cts.AccessibilityGlobalActionsTest.tearDown(AccessibilityGlobalActionsTest.java:97)\n ", + "data": null + } + ], + "testRunner": "cts", + "group": "", + "noBuild": false +} diff --git a/tests/data/cts/test_result.xml b/tests/data/cts/test_result.xml new file mode 100644 index 000000000..970f0a086 --- /dev/null +++ b/tests/data/cts/test_result.xml @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + java.lang.AssertionError: Unable to reach home screen + at org.junit.Assert.fail(Assert.java:89) + at + android.accessibilityservice.cts.utils.ActivityLaunchUtils.homeScreenOrBust(ActivityLaunchUtils.java:161) + at + android.accessibilityservice.cts.AccessibilityGlobalActionsTest.tearDown(AccessibilityGlobalActionsTest.java:97) + + + + + + From a3a9349509321a514cbfbbcae5093f37587c8e7c Mon Sep 17 00:00:00 2001 From: Konboi Date: Fri, 7 Apr 2023 16:10:29 +0900 Subject: [PATCH 05/12] set help document --- launchable/test_runners/cts.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/launchable/test_runners/cts.py b/launchable/test_runners/cts.py index d6cf11acb..69a8f4a4c 100644 --- a/launchable/test_runners/cts.py +++ b/launchable/test_runners/cts.py @@ -102,6 +102,9 @@ def parse_func(p: str) -> Dict: @click.argument('reports', required=True, nargs=-1) @launchable.record.tests def record_tests(client, reports): + """ + Beta: Report test result that Compatibility Test Suite (CTS) produced + """ for r in reports: client.report(r) @@ -111,6 +114,9 @@ def record_tests(client, reports): @launchable.subset def subset(client): + """ + Beta: Produces test list from previous test sessions for Compatibility Test Suite (CTS) + """ if not client.is_get_tests_from_previous_sessions: click.echo(click.style( "ERROR: cts profile supports only Zero Input Subsetting (ZIS). Please use `--get-tests-from-previous-sessions` option for subsetting", # noqa E501 From 72858ad2f19c542a2490c82a133e0d925fefd62f Mon Sep 17 00:00:00 2001 From: Konboi Date: Fri, 7 Apr 2023 16:11:18 +0900 Subject: [PATCH 06/12] add unit test --- tests/test_runners/test_cts.py | 92 ++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 tests/test_runners/test_cts.py diff --git a/tests/test_runners/test_cts.py b/tests/test_runners/test_cts.py new file mode 100644 index 000000000..41821a5bb --- /dev/null +++ b/tests/test_runners/test_cts.py @@ -0,0 +1,92 @@ +import gzip +import json +import os +from pathlib import Path +from unittest import mock + +import responses + +from launchable.utils.http_client import get_base_url +from tests.cli_test_case import CliTestCase + + +class CtsTest(CliTestCase): + test_files_dir = Path(__file__).parent.joinpath('../data/cts/').resolve() + result_file_path = test_files_dir.joinpath('record_test_result.json') + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_subset(self): + # cts is required using `--get-tests-from-previous-sessions` option + result = self.cli("subset", "--target", "30%", "--session", self.session, "cts") + self.assertEqual(result.exit_code, 1) + + mock_response = { + "testPaths": [ + [{"type": "Module", "name": "Cts1"}, {"type": "TestCase", "name": "android.example.package1"}], + [{"type": "Module", "name": "Cts2"}, {"type": "TestCase", "name": "android.example.package2"}], + ], + "testRunner": "cts", + "rest": [ + [{"type": "Module", "name": "Cts3"}, {"type": "TestCase", "name": "android.example.package3"}], + [{"type": "Module", "name": "Cts4"}, {"type": "TestCase", "name": "android.example.package4"}], + [{"type": "Module", "name": "Cts5"}, {"type": "TestCase", "name": "android.example.package5"}], + ], + "subsettingId": 123, + "summary": { + "subset": {"duration": 30, "candidates": 2, "rate": 30}, + "rest": {"duration": 70, "candidates": 3, "rate": 70} + }, + "isObservation": False, + } + + responses.replace(responses.POST, "{}/intake/organizations/{}/workspaces/{}/subset".format( + get_base_url(), + self.organization, + self.workspace), + json=mock_response, + status=200) + + result = self.cli( + "subset", + "--target", + "30%", + "--session", + self.session, + "--get-tests-from-previous-sessions", + "cts", + mix_stderr=False) + + self.assertEqual(result.exit_code, 0) + self.assertEqual(result.stdout, + '--include-filter "Cts1 android.example.package1" --include-filter "Cts2 android.example.package2"\n') + + result = self.cli( + "subset", + "--target", + "30%", + "--session", + self.session, + "--get-tests-from-previous-sessions", + "--output-exclusion-rules", + "cts", + mix_stderr=False) + + self.assertEqual(result.exit_code, 0) + self.assertEqual( + result.stdout, + '--exclude-filter "Cts3 android.example.package3" --exclude-filter "Cts4 android.example.package4" --exclude-filter "Cts5 android.example.package5"\n') # noqa E501 + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_record_tests(self): + result = self.cli('record', 'tests', '--session', self.session, + 'cts', str(self.test_files_dir) + "/test_result.xml") + self.assertEqual(result.exit_code, 0) + + payload = json.loads(gzip.decompress(responses.calls[1].request.body).decode()) + for e in payload["events"]: + e.pop("created_at", "") + + expected = self.load_json_from_file(self.result_file_path) + self.assert_json_orderless_equal(expected, payload) From 61702c3e42358ebab4e6cd4b499e6a50701d49e5 Mon Sep 17 00:00:00 2001 From: Konboi Date: Fri, 7 Apr 2023 17:08:22 +0900 Subject: [PATCH 07/12] fix type lint --- launchable/test_runners/cts.py | 70 +++++++++++++++++++++------------- 1 file changed, 43 insertions(+), 27 deletions(-) diff --git a/launchable/test_runners/cts.py b/launchable/test_runners/cts.py index 69a8f4a4c..82d0c2626 100644 --- a/launchable/test_runners/cts.py +++ b/launchable/test_runners/cts.py @@ -1,4 +1,5 @@ -from ast import Dict + +from typing import List from xml.etree import ElementTree as ET import click @@ -17,7 +18,7 @@ def build_path(module: str, test_case: str, test: str): ] -def parse_func(p: str) -> Dict: +def parse_func(p: str): """ # noqa: E501 # sample report format success case: @@ -44,56 +45,71 @@ def parse_func(p: str) -> Dict: """ + class TestResult: + def __init__(self, test_case_name: str, test_name: str, result: str, stdout: str, stderr: str): + self.test_case_name = test_case_name + self.test_name = test_name + self.result = result + self.stdout = stdout + self.stderr = stderr + + def case_event_status(self): + if self.result == "fail": + return CaseEvent.TEST_FAILED + elif self.result == "ASSUMPTION_FAILURE" or self.result == "IGNORED": + return CaseEvent.TEST_SKIPPED + + return CaseEvent.TEST_PASSED + events = [] tree = ET.parse(p) for module in tree.iter('Module'): - test_results = [] - total_duration = module.get("runtime") - module_name = module.get("name") + test_results: List[TestResult] = [] + total_duration = module.get("runtime", "0") + module_name = module.get("name", "") for test_case in module.iter("TestCase"): - test_case_name = test_case.get('name') + test_case_name = test_case.get('name', "") for test in test_case.iter("Test"): - result = test.get('result') - test_name = test.get("name") + result = test.get('result', "") + test_name = test.get("name", "") stdout = "" stderr = "" failure = test.find('Failure') if failure: - stack_trace = failure.find("StackTrace").text if failure.find("StackTrace") is not None else "" + stack_trace = "" + stack_trace_element = failure.find("StackTrace") + if stack_trace_element is not None: + stack_trace = str(stack_trace_element.text) - stdout = failure.get("message") + stdout = failure.get("message", "") stderr = stack_trace - test_results.append({ - "test_case_name": test_case_name, - "test_name": test_name, - "result": result, - "stdout": stdout, - "stderr": stderr, - }) + test_results.append(TestResult( + test_case_name=test_case_name, + test_name=test_name, + result=result, + stdout=stdout, + stderr=stderr)) if len(test_results) == 0: continue test_duration_msec_per_test = int(total_duration) / len(test_results) - for result in test_results: - status = CaseEvent.TEST_PASSED - if result.get("result") == "fail": - status = CaseEvent.TEST_FAILED - elif result.get("result") == "ASSUMPTION_FAILURE" or result.get("result") == "IGNORED": - status = CaseEvent.TEST_SKIPPED + for test_result in test_results: # type: TestResult + if module_name == "" or test_result.test_case_name == "" or test_result.test_name == "": + continue events.append(CaseEvent.create( - test_path=build_path(module_name, result.get("test_case_name"), result.get("test_name")), + test_path=build_path(module_name, test_result.test_case_name, test_result.test_name), duration_secs=float(test_duration_msec_per_test / 1000), - status=status, - stdout=result.get("stdout"), - stderr=result.get("stderr"), + status=test_result.case_event_status(), + stdout=test_result.stdout, + stderr=test_result.stderr, )) return (x for x in events) From 07a64a9c4fa13d704a9b923ab587e71024b807a5 Mon Sep 17 00:00:00 2001 From: Konboi Date: Fri, 7 Apr 2023 17:20:14 +0900 Subject: [PATCH 08/12] mend --- launchable/test_runners/cts.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/launchable/test_runners/cts.py b/launchable/test_runners/cts.py index 82d0c2626..c0b90e1a3 100644 --- a/launchable/test_runners/cts.py +++ b/launchable/test_runners/cts.py @@ -1,5 +1,3 @@ - -from typing import List from xml.etree import ElementTree as ET import click @@ -65,7 +63,7 @@ def case_event_status(self): tree = ET.parse(p) for module in tree.iter('Module'): - test_results: List[TestResult] = [] + test_results = [] total_duration = module.get("runtime", "0") module_name = module.get("name", "") From ebdf7da2b998556b8b5eb1c7725d65881887ce2d Mon Sep 17 00:00:00 2001 From: Konboi Date: Wed, 12 Apr 2023 17:16:25 +0900 Subject: [PATCH 09/12] fix subset altitude level from class to module --- launchable/test_runners/cts.py | 95 +++++++++++++++++-------------- tests/data/cts/subset_result.json | 38 +++++++++++++ tests/test_runners/test_cts.py | 47 +++++++++------ 3 files changed, 121 insertions(+), 59 deletions(-) create mode 100644 tests/data/cts/subset_result.json diff --git a/launchable/test_runners/cts.py b/launchable/test_runners/cts.py index c0b90e1a3..f63022e2d 100644 --- a/launchable/test_runners/cts.py +++ b/launchable/test_runners/cts.py @@ -3,17 +3,12 @@ import click from launchable.commands.record.case_event import CaseEvent -from launchable.testpath import TestPath from . import launchable - -def build_path(module: str, test_case: str, test: str): - return [ - {"type": "Module", "name": module}, - {"type": "TestCase", "name": test_case}, - {"type": "Test", "name": test}, - ] +# https://source.android.com/docs/compatibility/cts/command-console-v2 +include_option = "--include-filter" +exclude_option = "--exclude-filter" def parse_func(p: str): @@ -59,6 +54,13 @@ def case_event_status(self): return CaseEvent.TEST_PASSED + def build_record_test_path(module: str, test_case: str, test: str): + return [ + {"type": "Module", "name": module}, + {"type": "TestCase", "name": test_case}, + {"type": "Test", "name": test}, + ] + events = [] tree = ET.parse(p) @@ -103,7 +105,7 @@ def case_event_status(self): continue events.append(CaseEvent.create( - test_path=build_path(module_name, test_result.test_case_name, test_result.test_name), + test_path=build_record_test_path(module_name, test_result.test_case_name, test_result.test_name), duration_secs=float(test_duration_msec_per_test / 1000), status=test_result.case_event_status(), stdout=test_result.stdout, @@ -131,37 +133,46 @@ def subset(client): """ Beta: Produces test list from previous test sessions for Compatibility Test Suite (CTS) """ - if not client.is_get_tests_from_previous_sessions: - click.echo(click.style( - "ERROR: cts profile supports only Zero Input Subsetting (ZIS). Please use `--get-tests-from-previous-sessions` option for subsetting", # noqa E501 - fg="red"), - err=True) - exit(1) - - include_option = "--include-filter" - exclude_option = "--exclude-filter" - - def formatter(test_path: TestPath): - module = "" - test_case = "" - for path in test_path: - t = path.get('type', '') - n = path.get('name', '') - - if t == "Module": - module = n - elif t == "TestCase": - test_case = n - - if module == "" or test_case == "": - return - - option = include_option - if client.is_output_exclusion_rules: - option = exclude_option - - return "{option} \"{module} {test_case}\"".format(option=option, module=module, test_case=test_case) - - client.formatter = formatter - client.separator = " " + start_module = False + + """ # noqa: E501 + # This is sample output for `cts-tf > list modules` + ================== + Notice: + We collect anonymous usage statistics in accordance with our Content Licenses (https://source.android.com/setup/start/licenses), Contributor License Agreement (https://opensource.google.com/docs/cla/), Privacy Policy (https://policies.google.com/privacy) and Terms of Service (https://policies.google.com/terms). + ================== + Android Compatibility Test Suite 12.1_r5 (9566553) + Use "help" or "help all" to get more information on running commands. + Non-interactive mode: Running initial command then exiting. + Using commandline arguments as starting command: [list, modules] + arm64-v8a CtsAbiOverrideHostTestCases[instant] + arm64-v8a CtsAbiOverrideHostTestCases[secondary_user] + arm64-v8a CtsAbiOverrideHostTestCases + armeabi-v7a CtsAbiOverrideHostTestCases + """ + + for t in client.stdin(): + if "starting command" in t: + start_module = True + continue + + if not start_module: + continue + + lines = t.rstrip("\n").split() + if len(lines) != 2: + click.echo( + click.style( + "Warning: {line} is not expected Module format and skipped".format( + line=t), + fg="yellow"), + err=True) + continue + client.test_path([{"type": "Module", "name": lines[1]}]) + + option = include_option + if client.is_output_exclusion_rules: + option = exclude_option + + client.formatter = lambda x: "{option} \"{module}\"".format(option=option, module=x[0]['name']) client.run() diff --git a/tests/data/cts/subset_result.json b/tests/data/cts/subset_result.json new file mode 100644 index 000000000..ac2aea098 --- /dev/null +++ b/tests/data/cts/subset_result.json @@ -0,0 +1,38 @@ +{ + "testPaths": [ + [ + { + "type": "Module", + "name": "CtsAbiOverrideHostTestCases[instant]" + } + ], + [ + { + "type": "Module", + "name": "CtsAbiOverrideHostTestCases[secondary_user]" + } + ], + [ + { + "type": "Module", + "name": "CtsAbiOverrideHostTestCases" + } + ], + [ + { + "type": "Module", + "name": "CtsAbiOverrideHostTestCases" + } + ] + ], + "testRunner": "cts", + "session": { + "id": "16" + }, + "ignoreNewTests": false, + "getTestsFromPreviousSessions": false, + "goal": { + "type": "subset-by-percentage", + "percentage": 0.3 + } +} diff --git a/tests/test_runners/test_cts.py b/tests/test_runners/test_cts.py index 41821a5bb..249c3159e 100644 --- a/tests/test_runners/test_cts.py +++ b/tests/test_runners/test_cts.py @@ -13,29 +13,41 @@ class CtsTest(CliTestCase): test_files_dir = Path(__file__).parent.joinpath('../data/cts/').resolve() result_file_path = test_files_dir.joinpath('record_test_result.json') + subset_result_test_path = test_files_dir.joinpath('subset_result.json') @responses.activate @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) def test_subset(self): - # cts is required using `--get-tests-from-previous-sessions` option - result = self.cli("subset", "--target", "30%", "--session", self.session, "cts") - self.assertEqual(result.exit_code, 1) + pipe = """ # noqa: E501 +================== +Notice: +We collect anonymous usage statistics in accordance with our Content Licenses (https://source.android.com/setup/start/licenses), Contributor License Agreement (https://opensource.google.com/docs/cla/), Privacy Policy (https://policies.google.com/privacy) and Terms of Service (https://policies.google.com/terms). +================== +Android Compatibility Test Suite 12.1_r5 (9566553) +Use "help" or "help all" to get more information on running commands. +Non-interactive mode: Running initial command then exiting. +Using commandline arguments as starting command: [list, modules] +arm64-v8a CtsAbiOverrideHostTestCases[instant] +arm64-v8a CtsAbiOverrideHostTestCases[secondary_user] +arm64-v8a CtsAbiOverrideHostTestCases +armeabi-v7a CtsAbiOverrideHostTestCases + """ mock_response = { "testPaths": [ - [{"type": "Module", "name": "Cts1"}, {"type": "TestCase", "name": "android.example.package1"}], - [{"type": "Module", "name": "Cts2"}, {"type": "TestCase", "name": "android.example.package2"}], + [{"type": "Module", "name": "CtsAbiOverrideHostTestCases[instant]"}], + [{"type": "Module", "name": "CtsAbiOverrideHostTestCases"}], + ], "testRunner": "cts", "rest": [ - [{"type": "Module", "name": "Cts3"}, {"type": "TestCase", "name": "android.example.package3"}], - [{"type": "Module", "name": "Cts4"}, {"type": "TestCase", "name": "android.example.package4"}], - [{"type": "Module", "name": "Cts5"}, {"type": "TestCase", "name": "android.example.package5"}], + [{"type": "Module", "name": "CtsAbiOverrideHostTestCases[secondary_user]"}], + [{"type": "Module", "name": "CtsAbiOverrideHostTestCases"}], ], "subsettingId": 123, "summary": { "subset": {"duration": 30, "candidates": 2, "rate": 30}, - "rest": {"duration": 70, "candidates": 3, "rate": 70} + "rest": {"duration": 70, "candidates": 2, "rate": 70} }, "isObservation": False, } @@ -53,13 +65,15 @@ def test_subset(self): "30%", "--session", self.session, - "--get-tests-from-previous-sessions", "cts", + input=pipe, mix_stderr=False) - self.assertEqual(result.exit_code, 0) - self.assertEqual(result.stdout, - '--include-filter "Cts1 android.example.package1" --include-filter "Cts2 android.example.package2"\n') + output = "--include-filter \"CtsAbiOverrideHostTestCases[instant]\"\n--include-filter \"CtsAbiOverrideHostTestCases\"\n" + self.assertEqual(output, result.stdout) + payload = json.loads(gzip.decompress(responses.calls[0].request.body).decode()) + expected = self.load_json_from_file(self.test_files_dir.joinpath('subset_result.json')) + self.assert_json_orderless_equal(expected, payload) result = self.cli( "subset", @@ -67,15 +81,14 @@ def test_subset(self): "30%", "--session", self.session, - "--get-tests-from-previous-sessions", "--output-exclusion-rules", "cts", + input=pipe, mix_stderr=False) self.assertEqual(result.exit_code, 0) - self.assertEqual( - result.stdout, - '--exclude-filter "Cts3 android.example.package3" --exclude-filter "Cts4 android.example.package4" --exclude-filter "Cts5 android.example.package5"\n') # noqa E501 + output = "--exclude-filter \"CtsAbiOverrideHostTestCases[secondary_user]\"\n--exclude-filter \"CtsAbiOverrideHostTestCases\"\n" # noqa: E501 + self.assertEqual(output, result.stdout) @responses.activate @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) From abd02895bcab4d3ad9f0b061a35363dde15b974c Mon Sep 17 00:00:00 2001 From: Konboi Date: Wed, 12 Apr 2023 17:22:10 +0900 Subject: [PATCH 10/12] fix comment --- launchable/test_runners/cts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launchable/test_runners/cts.py b/launchable/test_runners/cts.py index f63022e2d..082a88253 100644 --- a/launchable/test_runners/cts.py +++ b/launchable/test_runners/cts.py @@ -136,7 +136,7 @@ def subset(client): start_module = False """ # noqa: E501 - # This is sample output for `cts-tf > list modules` + # This is sample output of `cts-tradefed list modules` ================== Notice: We collect anonymous usage statistics in accordance with our Content Licenses (https://source.android.com/setup/start/licenses), Contributor License Agreement (https://opensource.google.com/docs/cla/), Privacy Policy (https://policies.google.com/privacy) and Terms of Service (https://policies.google.com/terms). From 6ed582deed125df52edf3850dba91fbefe9089e4 Mon Sep 17 00:00:00 2001 From: Konboi Date: Thu, 13 Apr 2023 10:30:05 +0900 Subject: [PATCH 11/12] fix value name and add comment of line --- launchable/test_runners/cts.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/launchable/test_runners/cts.py b/launchable/test_runners/cts.py index 082a88253..d9f3c37ae 100644 --- a/launchable/test_runners/cts.py +++ b/launchable/test_runners/cts.py @@ -159,8 +159,9 @@ def subset(client): if not start_module: continue - lines = t.rstrip("\n").split() - if len(lines) != 2: + # e.g) armeabi-v7a CtsAbiOverrideHostTestCases + device_and_module = t.rstrip("\n").split() + if len(device_and_module) != 2: click.echo( click.style( "Warning: {line} is not expected Module format and skipped".format( @@ -168,7 +169,8 @@ def subset(client): fg="yellow"), err=True) continue - client.test_path([{"type": "Module", "name": lines[1]}]) + + client.test_path([{"type": "Module", "name": device_and_module[1]}]) option = include_option if client.is_output_exclusion_rules: From 87ef7ac4172db8b28df2964c06343dd6cc637878 Mon Sep 17 00:00:00 2001 From: Konboi Date: Thu, 13 Apr 2023 10:33:25 +0900 Subject: [PATCH 12/12] add comments --- launchable/test_runners/cts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/launchable/test_runners/cts.py b/launchable/test_runners/cts.py index d9f3c37ae..a715f1c92 100644 --- a/launchable/test_runners/cts.py +++ b/launchable/test_runners/cts.py @@ -119,7 +119,7 @@ def build_record_test_path(module: str, test_case: str, test: str): @launchable.record.tests def record_tests(client, reports): """ - Beta: Report test result that Compatibility Test Suite (CTS) produced + Beta: Report test result that Compatibility Test Suite (CTS) produced. Supports only CTS v2 """ for r in reports: client.report(r) @@ -131,7 +131,7 @@ def record_tests(client, reports): @launchable.subset def subset(client): """ - Beta: Produces test list from previous test sessions for Compatibility Test Suite (CTS) + Beta: Produces test list from previous test sessions for Compatibility Test Suite (CTS). Supports only CTS v2 """ start_module = False